LCOV - code coverage report
Current view: top level - lib/blobfs - blobfs.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 1099 1481 74.2 %
Date: 2024-07-12 03:48:00 Functions: 114 132 86.4 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2017 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  */
       5             : 
       6             : #include "spdk/stdinc.h"
       7             : 
       8             : #include "spdk/blobfs.h"
       9             : #include "cache_tree.h"
      10             : 
      11             : #include "spdk/queue.h"
      12             : #include "spdk/thread.h"
      13             : #include "spdk/assert.h"
      14             : #include "spdk/env.h"
      15             : #include "spdk/util.h"
      16             : #include "spdk/log.h"
      17             : #include "spdk/trace.h"
      18             : 
      19             : #include "spdk_internal/trace_defs.h"
      20             : 
      21             : #define BLOBFS_TRACE(file, str, args...) \
      22             :         SPDK_DEBUGLOG(blobfs, "file=%s " str, file->name, ##args)
      23             : 
      24             : #define BLOBFS_TRACE_RW(file, str, args...) \
      25             :         SPDK_DEBUGLOG(blobfs_rw, "file=%s " str, file->name, ##args)
      26             : 
      27             : #define BLOBFS_DEFAULT_CACHE_SIZE (4ULL * 1024 * 1024 * 1024)
      28             : #define SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ (1024 * 1024)
      29             : 
      30             : #define SPDK_BLOBFS_SIGNATURE   "BLOBFS"
      31             : 
      32             : static uint64_t g_fs_cache_size = BLOBFS_DEFAULT_CACHE_SIZE;
      33             : static struct spdk_mempool *g_cache_pool;
      34             : static TAILQ_HEAD(, spdk_file) g_caches = TAILQ_HEAD_INITIALIZER(g_caches);
      35             : static struct spdk_poller *g_cache_pool_mgmt_poller;
      36             : static struct spdk_thread *g_cache_pool_thread;
      37             : #define BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US 1000ULL
      38             : static int g_fs_count = 0;
      39             : static pthread_mutex_t g_cache_init_lock = PTHREAD_MUTEX_INITIALIZER;
      40             : 
      41           2 : SPDK_TRACE_REGISTER_FN(blobfs_trace, "blobfs", TRACE_GROUP_BLOBFS)
      42             : {
      43           0 :         struct spdk_trace_tpoint_opts opts[] = {
      44             :                 {
      45             :                         "BLOBFS_XATTR_START", TRACE_BLOBFS_XATTR_START,
      46             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      47             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      48             :                 },
      49             :                 {
      50             :                         "BLOBFS_XATTR_END", TRACE_BLOBFS_XATTR_END,
      51             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      52             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      53             :                 },
      54             :                 {
      55             :                         "BLOBFS_OPEN", TRACE_BLOBFS_OPEN,
      56             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      57             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      58             :                 },
      59             :                 {
      60             :                         "BLOBFS_CLOSE", TRACE_BLOBFS_CLOSE,
      61             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      62             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      63             :                 },
      64             :                 {
      65             :                         "BLOBFS_DELETE_START", TRACE_BLOBFS_DELETE_START,
      66             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      67             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      68             :                 },
      69             :                 {
      70             :                         "BLOBFS_DELETE_DONE", TRACE_BLOBFS_DELETE_DONE,
      71             :                         OWNER_TYPE_NONE, OBJECT_NONE, 0,
      72             :                         {{ "file", SPDK_TRACE_ARG_TYPE_STR, 40 }},
      73             :                 }
      74             :         };
      75             : 
      76           0 :         spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
      77           0 : }
      78             : 
      79             : void
      80          13 : cache_buffer_free(struct cache_buffer *cache_buffer)
      81             : {
      82          13 :         spdk_mempool_put(g_cache_pool, cache_buffer->buf);
      83          13 :         free(cache_buffer);
      84          13 : }
      85             : 
      86             : #define CACHE_READAHEAD_THRESHOLD       (128 * 1024)
      87             : 
      88             : struct spdk_file {
      89             :         struct spdk_filesystem  *fs;
      90             :         struct spdk_blob        *blob;
      91             :         char                    *name;
      92             :         uint64_t                length;
      93             :         bool                    is_deleted;
      94             :         bool                    open_for_writing;
      95             :         uint64_t                length_flushed;
      96             :         uint64_t                length_xattr;
      97             :         uint64_t                append_pos;
      98             :         uint64_t                seq_byte_count;
      99             :         uint64_t                next_seq_offset;
     100             :         uint32_t                priority;
     101             :         TAILQ_ENTRY(spdk_file)  tailq;
     102             :         spdk_blob_id            blobid;
     103             :         uint32_t                ref_count;
     104             :         pthread_spinlock_t      lock;
     105             :         struct cache_buffer     *last;
     106             :         struct cache_tree       *tree;
     107             :         TAILQ_HEAD(open_requests_head, spdk_fs_request) open_requests;
     108             :         TAILQ_HEAD(sync_requests_head, spdk_fs_request) sync_requests;
     109             :         TAILQ_ENTRY(spdk_file)  cache_tailq;
     110             : };
     111             : 
     112             : struct spdk_deleted_file {
     113             :         spdk_blob_id    id;
     114             :         TAILQ_ENTRY(spdk_deleted_file)  tailq;
     115             : };
     116             : 
     117             : struct spdk_filesystem {
     118             :         struct spdk_blob_store  *bs;
     119             :         TAILQ_HEAD(, spdk_file) files;
     120             :         struct spdk_bs_opts     bs_opts;
     121             :         struct spdk_bs_dev      *bdev;
     122             :         fs_send_request_fn      send_request;
     123             : 
     124             :         struct {
     125             :                 uint32_t                max_ops;
     126             :                 struct spdk_io_channel  *sync_io_channel;
     127             :                 struct spdk_fs_channel  *sync_fs_channel;
     128             :         } sync_target;
     129             : 
     130             :         struct {
     131             :                 uint32_t                max_ops;
     132             :                 struct spdk_io_channel  *md_io_channel;
     133             :                 struct spdk_fs_channel  *md_fs_channel;
     134             :         } md_target;
     135             : 
     136             :         struct {
     137             :                 uint32_t                max_ops;
     138             :         } io_target;
     139             : };
     140             : 
     141             : struct spdk_fs_cb_args {
     142             :         union {
     143             :                 spdk_fs_op_with_handle_complete         fs_op_with_handle;
     144             :                 spdk_fs_op_complete                     fs_op;
     145             :                 spdk_file_op_with_handle_complete       file_op_with_handle;
     146             :                 spdk_file_op_complete                   file_op;
     147             :                 spdk_file_stat_op_complete              stat_op;
     148             :         } fn;
     149             :         void *arg;
     150             :         sem_t *sem;
     151             :         struct spdk_filesystem *fs;
     152             :         struct spdk_file *file;
     153             :         int rc;
     154             :         int *rwerrno;
     155             :         struct iovec *iovs;
     156             :         uint32_t iovcnt;
     157             :         struct iovec iov;
     158             :         union {
     159             :                 struct {
     160             :                         TAILQ_HEAD(, spdk_deleted_file) deleted_files;
     161             :                 } fs_load;
     162             :                 struct {
     163             :                         uint64_t        length;
     164             :                 } truncate;
     165             :                 struct {
     166             :                         struct spdk_io_channel  *channel;
     167             :                         void            *pin_buf;
     168             :                         int             is_read;
     169             :                         off_t           offset;
     170             :                         size_t          length;
     171             :                         uint64_t        start_lba;
     172             :                         uint64_t        num_lba;
     173             :                         uint32_t        blocklen;
     174             :                 } rw;
     175             :                 struct {
     176             :                         const char      *old_name;
     177             :                         const char      *new_name;
     178             :                 } rename;
     179             :                 struct {
     180             :                         struct cache_buffer     *cache_buffer;
     181             :                         uint64_t                length;
     182             :                 } flush;
     183             :                 struct {
     184             :                         struct cache_buffer     *cache_buffer;
     185             :                         uint64_t                length;
     186             :                         uint64_t                offset;
     187             :                 } readahead;
     188             :                 struct {
     189             :                         /* offset of the file when the sync request was made */
     190             :                         uint64_t                        offset;
     191             :                         TAILQ_ENTRY(spdk_fs_request)    tailq;
     192             :                         bool                            xattr_in_progress;
     193             :                         /* length written to the xattr for this file - this should
     194             :                          * always be the same as the offset if only one thread is
     195             :                          * writing to the file, but could differ if multiple threads
     196             :                          * are appending
     197             :                          */
     198             :                         uint64_t                        length;
     199             :                 } sync;
     200             :                 struct {
     201             :                         uint32_t                        num_clusters;
     202             :                 } resize;
     203             :                 struct {
     204             :                         const char      *name;
     205             :                         uint32_t        flags;
     206             :                         TAILQ_ENTRY(spdk_fs_request)    tailq;
     207             :                 } open;
     208             :                 struct {
     209             :                         const char              *name;
     210             :                         struct spdk_blob        *blob;
     211             :                 } create;
     212             :                 struct {
     213             :                         const char      *name;
     214             :                 } delete;
     215             :                 struct {
     216             :                         const char      *name;
     217             :                 } stat;
     218             :         } op;
     219             : };
     220             : 
     221             : static void file_free(struct spdk_file *file);
     222             : static void fs_io_device_unregister(struct spdk_filesystem *fs);
     223             : static void fs_free_io_channels(struct spdk_filesystem *fs);
     224             : 
     225             : void
     226           0 : spdk_fs_opts_init(struct spdk_blobfs_opts *opts)
     227             : {
     228           0 :         opts->cluster_sz = SPDK_BLOBFS_DEFAULT_OPTS_CLUSTER_SZ;
     229           0 : }
     230             : 
     231             : static int _blobfs_cache_pool_reclaim(void *arg);
     232             : 
     233             : static bool
     234           0 : blobfs_cache_pool_need_reclaim(void)
     235             : {
     236             :         size_t count;
     237             : 
     238           0 :         count = spdk_mempool_count(g_cache_pool);
     239             :         /* We define a aggressive policy here as the requirements from db_bench are batched, so start the poller
     240             :          *  when the number of available cache buffer is less than 1/5 of total buffers.
     241             :          */
     242           0 :         if (count > (size_t)g_fs_cache_size / CACHE_BUFFER_SIZE / 5) {
     243           0 :                 return false;
     244             :         }
     245             : 
     246           0 :         return true;
     247             : }
     248             : 
     249             : static void
     250          20 : __start_cache_pool_mgmt(void *ctx)
     251             : {
     252          20 :         assert(g_cache_pool_mgmt_poller == NULL);
     253          20 :         g_cache_pool_mgmt_poller = SPDK_POLLER_REGISTER(_blobfs_cache_pool_reclaim, NULL,
     254             :                                    BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
     255          20 : }
     256             : 
     257             : static void
     258          20 : __stop_cache_pool_mgmt(void *ctx)
     259             : {
     260          20 :         spdk_poller_unregister(&g_cache_pool_mgmt_poller);
     261             : 
     262          20 :         assert(g_cache_pool != NULL);
     263          20 :         assert(spdk_mempool_count(g_cache_pool) == g_fs_cache_size / CACHE_BUFFER_SIZE);
     264          20 :         spdk_mempool_free(g_cache_pool);
     265          20 :         g_cache_pool = NULL;
     266             : 
     267          20 :         spdk_thread_exit(g_cache_pool_thread);
     268          20 : }
     269             : 
     270             : static void
     271          20 : allocate_cache_pool(void)
     272             : {
     273          20 :         assert(g_cache_pool == NULL);
     274          20 :         g_cache_pool = spdk_mempool_create("spdk_fs_cache",
     275             :                                            g_fs_cache_size / CACHE_BUFFER_SIZE,
     276             :                                            CACHE_BUFFER_SIZE,
     277             :                                            SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
     278             :                                            SPDK_ENV_SOCKET_ID_ANY);
     279          20 :         if (!g_cache_pool) {
     280           0 :                 if (spdk_mempool_lookup("spdk_fs_cache") != NULL) {
     281           0 :                         SPDK_ERRLOG("Unable to allocate mempool: already exists\n");
     282           0 :                         SPDK_ERRLOG("Probably running in multiprocess environment, which is "
     283             :                                     "unsupported by the blobfs library\n");
     284             :                 } else {
     285           0 :                         SPDK_ERRLOG("Create mempool failed, you may "
     286             :                                     "increase the memory and try again\n");
     287             :                 }
     288           0 :                 assert(false);
     289             :         }
     290          20 : }
     291             : 
     292             : static void
     293          20 : initialize_global_cache(void)
     294             : {
     295          20 :         pthread_mutex_lock(&g_cache_init_lock);
     296          20 :         if (g_fs_count == 0) {
     297          20 :                 allocate_cache_pool();
     298          20 :                 g_cache_pool_thread = spdk_thread_create("cache_pool_mgmt", NULL);
     299          20 :                 assert(g_cache_pool_thread != NULL);
     300          20 :                 spdk_thread_send_msg(g_cache_pool_thread, __start_cache_pool_mgmt, NULL);
     301             :         }
     302          20 :         g_fs_count++;
     303          20 :         pthread_mutex_unlock(&g_cache_init_lock);
     304          20 : }
     305             : 
     306             : static void
     307          20 : free_global_cache(void)
     308             : {
     309          20 :         pthread_mutex_lock(&g_cache_init_lock);
     310          20 :         g_fs_count--;
     311          20 :         if (g_fs_count == 0) {
     312          20 :                 spdk_thread_send_msg(g_cache_pool_thread, __stop_cache_pool_mgmt, NULL);
     313             :         }
     314          20 :         pthread_mutex_unlock(&g_cache_init_lock);
     315          20 : }
     316             : 
     317             : static uint64_t
     318          21 : __file_get_blob_size(struct spdk_file *file)
     319             : {
     320             :         uint64_t cluster_sz;
     321             : 
     322          21 :         cluster_sz = file->fs->bs_opts.cluster_sz;
     323          21 :         return cluster_sz * spdk_blob_get_num_clusters(file->blob);
     324             : }
     325             : 
     326             : struct spdk_fs_request {
     327             :         struct spdk_fs_cb_args          args;
     328             :         TAILQ_ENTRY(spdk_fs_request)    link;
     329             :         struct spdk_fs_channel          *channel;
     330             : };
     331             : 
     332             : struct spdk_fs_channel {
     333             :         struct spdk_fs_request          *req_mem;
     334             :         TAILQ_HEAD(, spdk_fs_request)   reqs;
     335             :         sem_t                           sem;
     336             :         struct spdk_filesystem          *fs;
     337             :         struct spdk_io_channel          *bs_channel;
     338             :         fs_send_request_fn              send_request;
     339             :         bool                            sync;
     340             :         uint32_t                        outstanding_reqs;
     341             :         pthread_spinlock_t              lock;
     342             : };
     343             : 
     344             : /* For now, this is effectively an alias. But eventually we'll shift
     345             :  * some data members over. */
     346             : struct spdk_fs_thread_ctx {
     347             :         struct spdk_fs_channel  ch;
     348             : };
     349             : 
     350             : static struct spdk_fs_request *
     351         156 : alloc_fs_request_with_iov(struct spdk_fs_channel *channel, uint32_t iovcnt)
     352             : {
     353             :         struct spdk_fs_request *req;
     354         156 :         struct iovec *iovs = NULL;
     355             : 
     356         156 :         if (iovcnt > 1) {
     357           4 :                 iovs = calloc(iovcnt, sizeof(struct iovec));
     358           4 :                 if (!iovs) {
     359           0 :                         return NULL;
     360             :                 }
     361             :         }
     362             : 
     363         156 :         if (channel->sync) {
     364          66 :                 pthread_spin_lock(&channel->lock);
     365             :         }
     366             : 
     367         156 :         req = TAILQ_FIRST(&channel->reqs);
     368         156 :         if (req) {
     369         156 :                 channel->outstanding_reqs++;
     370         156 :                 TAILQ_REMOVE(&channel->reqs, req, link);
     371             :         }
     372             : 
     373         156 :         if (channel->sync) {
     374          66 :                 pthread_spin_unlock(&channel->lock);
     375             :         }
     376             : 
     377         156 :         if (req == NULL) {
     378           0 :                 SPDK_ERRLOG("Cannot allocate req on spdk_fs_channel =%p\n", channel);
     379           0 :                 free(iovs);
     380           0 :                 return NULL;
     381             :         }
     382         156 :         memset(req, 0, sizeof(*req));
     383         156 :         req->channel = channel;
     384         156 :         if (iovcnt > 1) {
     385           4 :                 req->args.iovs = iovs;
     386             :         } else {
     387         152 :                 req->args.iovs = &req->args.iov;
     388             :         }
     389         156 :         req->args.iovcnt = iovcnt;
     390             : 
     391         156 :         return req;
     392             : }
     393             : 
     394             : static struct spdk_fs_request *
     395         148 : alloc_fs_request(struct spdk_fs_channel *channel)
     396             : {
     397         148 :         return alloc_fs_request_with_iov(channel, 0);
     398             : }
     399             : 
     400             : static void
     401         156 : free_fs_request(struct spdk_fs_request *req)
     402             : {
     403         156 :         struct spdk_fs_channel *channel = req->channel;
     404             : 
     405         156 :         if (req->args.iovcnt > 1) {
     406           4 :                 free(req->args.iovs);
     407             :         }
     408             : 
     409         156 :         if (channel->sync) {
     410          66 :                 pthread_spin_lock(&channel->lock);
     411             :         }
     412             : 
     413         156 :         TAILQ_INSERT_HEAD(&req->channel->reqs, req, link);
     414         156 :         channel->outstanding_reqs--;
     415             : 
     416         156 :         if (channel->sync) {
     417          66 :                 pthread_spin_unlock(&channel->lock);
     418             :         }
     419         156 : }
     420             : 
     421             : static int
     422          53 : fs_channel_create(struct spdk_filesystem *fs, struct spdk_fs_channel *channel,
     423             :                   uint32_t max_ops)
     424             : {
     425             :         uint32_t i;
     426             : 
     427          53 :         channel->req_mem = calloc(max_ops, sizeof(struct spdk_fs_request));
     428          53 :         if (!channel->req_mem) {
     429           0 :                 return -1;
     430             :         }
     431             : 
     432          53 :         channel->outstanding_reqs = 0;
     433          53 :         TAILQ_INIT(&channel->reqs);
     434          53 :         sem_init(&channel->sem, 0, 0);
     435             : 
     436       27189 :         for (i = 0; i < max_ops; i++) {
     437       27136 :                 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link);
     438             :         }
     439             : 
     440          53 :         channel->fs = fs;
     441             : 
     442          53 :         return 0;
     443             : }
     444             : 
     445             : static int
     446          20 : fs_md_channel_create(void *io_device, void *ctx_buf)
     447             : {
     448             :         struct spdk_filesystem          *fs;
     449          20 :         struct spdk_fs_channel          *channel = ctx_buf;
     450             : 
     451          20 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, md_target);
     452             : 
     453          20 :         return fs_channel_create(fs, channel, fs->md_target.max_ops);
     454             : }
     455             : 
     456             : static int
     457          20 : fs_sync_channel_create(void *io_device, void *ctx_buf)
     458             : {
     459             :         struct spdk_filesystem          *fs;
     460          20 :         struct spdk_fs_channel          *channel = ctx_buf;
     461             : 
     462          20 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, sync_target);
     463             : 
     464          20 :         return fs_channel_create(fs, channel, fs->sync_target.max_ops);
     465             : }
     466             : 
     467             : static int
     468           1 : fs_io_channel_create(void *io_device, void *ctx_buf)
     469             : {
     470             :         struct spdk_filesystem          *fs;
     471           1 :         struct spdk_fs_channel          *channel = ctx_buf;
     472             : 
     473           1 :         fs = SPDK_CONTAINEROF(io_device, struct spdk_filesystem, io_target);
     474             : 
     475           1 :         return fs_channel_create(fs, channel, fs->io_target.max_ops);
     476             : }
     477             : 
     478             : static void
     479          53 : fs_channel_destroy(void *io_device, void *ctx_buf)
     480             : {
     481          53 :         struct spdk_fs_channel *channel = ctx_buf;
     482             : 
     483          53 :         if (channel->outstanding_reqs > 0) {
     484           0 :                 SPDK_ERRLOG("channel freed with %" PRIu32 " outstanding requests!\n",
     485             :                             channel->outstanding_reqs);
     486             :         }
     487             : 
     488          53 :         free(channel->req_mem);
     489          53 :         if (channel->bs_channel != NULL) {
     490          41 :                 spdk_bs_free_io_channel(channel->bs_channel);
     491             :         }
     492          53 : }
     493             : 
     494             : static void
     495           0 : __send_request_direct(fs_request_fn fn, void *arg)
     496             : {
     497           0 :         fn(arg);
     498           0 : }
     499             : 
     500             : static void
     501          20 : common_fs_bs_init(struct spdk_filesystem *fs, struct spdk_blob_store *bs)
     502             : {
     503          20 :         fs->bs = bs;
     504          20 :         fs->bs_opts.cluster_sz = spdk_bs_get_cluster_size(bs);
     505          20 :         fs->md_target.md_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
     506          20 :         fs->md_target.md_fs_channel->send_request = __send_request_direct;
     507          20 :         fs->sync_target.sync_fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
     508          20 :         fs->sync_target.sync_fs_channel->send_request = __send_request_direct;
     509             : 
     510          20 :         initialize_global_cache();
     511          20 : }
     512             : 
     513             : static void
     514          18 : init_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
     515             : {
     516          18 :         struct spdk_fs_request *req = ctx;
     517          18 :         struct spdk_fs_cb_args *args = &req->args;
     518          18 :         struct spdk_filesystem *fs = args->fs;
     519             : 
     520          18 :         if (bserrno == 0) {
     521          18 :                 common_fs_bs_init(fs, bs);
     522             :         } else {
     523           0 :                 free(fs);
     524           0 :                 fs = NULL;
     525             :         }
     526             : 
     527          18 :         args->fn.fs_op_with_handle(args->arg, fs, bserrno);
     528          18 :         free_fs_request(req);
     529          18 : }
     530             : 
     531             : static struct spdk_filesystem *
     532          20 : fs_alloc(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn)
     533             : {
     534             :         struct spdk_filesystem *fs;
     535             : 
     536          20 :         fs = calloc(1, sizeof(*fs));
     537          20 :         if (fs == NULL) {
     538           0 :                 return NULL;
     539             :         }
     540             : 
     541          20 :         fs->bdev = dev;
     542          20 :         fs->send_request = send_request_fn;
     543          20 :         TAILQ_INIT(&fs->files);
     544             : 
     545          20 :         fs->md_target.max_ops = 512;
     546          20 :         spdk_io_device_register(&fs->md_target, fs_md_channel_create, fs_channel_destroy,
     547             :                                 sizeof(struct spdk_fs_channel), "blobfs_md");
     548          20 :         fs->md_target.md_io_channel = spdk_get_io_channel(&fs->md_target);
     549          20 :         fs->md_target.md_fs_channel = spdk_io_channel_get_ctx(fs->md_target.md_io_channel);
     550             : 
     551          20 :         fs->sync_target.max_ops = 512;
     552          20 :         spdk_io_device_register(&fs->sync_target, fs_sync_channel_create, fs_channel_destroy,
     553             :                                 sizeof(struct spdk_fs_channel), "blobfs_sync");
     554          20 :         fs->sync_target.sync_io_channel = spdk_get_io_channel(&fs->sync_target);
     555          20 :         fs->sync_target.sync_fs_channel = spdk_io_channel_get_ctx(fs->sync_target.sync_io_channel);
     556             : 
     557          20 :         fs->io_target.max_ops = 512;
     558          20 :         spdk_io_device_register(&fs->io_target, fs_io_channel_create, fs_channel_destroy,
     559             :                                 sizeof(struct spdk_fs_channel), "blobfs_io");
     560             : 
     561          20 :         return fs;
     562             : }
     563             : 
     564             : static void
     565          51 : __wake_caller(void *arg, int fserrno)
     566             : {
     567          51 :         struct spdk_fs_cb_args *args = arg;
     568             : 
     569          51 :         if ((args->rwerrno != NULL) && (*(args->rwerrno) == 0) && fserrno) {
     570           0 :                 *(args->rwerrno) = fserrno;
     571             :         }
     572          51 :         args->rc = fserrno;
     573          51 :         sem_post(args->sem);
     574          51 : }
     575             : 
     576             : void
     577          18 : spdk_fs_init(struct spdk_bs_dev *dev, struct spdk_blobfs_opts *opt,
     578             :              fs_send_request_fn send_request_fn,
     579             :              spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
     580             : {
     581             :         struct spdk_filesystem *fs;
     582             :         struct spdk_fs_request *req;
     583             :         struct spdk_fs_cb_args *args;
     584          18 :         struct spdk_bs_opts opts = {};
     585             : 
     586          18 :         fs = fs_alloc(dev, send_request_fn);
     587          18 :         if (fs == NULL) {
     588           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     589           0 :                 return;
     590             :         }
     591             : 
     592          18 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
     593          18 :         if (req == NULL) {
     594           0 :                 fs_free_io_channels(fs);
     595           0 :                 fs_io_device_unregister(fs);
     596           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     597           0 :                 return;
     598             :         }
     599             : 
     600          18 :         args = &req->args;
     601          18 :         args->fn.fs_op_with_handle = cb_fn;
     602          18 :         args->arg = cb_arg;
     603          18 :         args->fs = fs;
     604             : 
     605          18 :         spdk_bs_opts_init(&opts, sizeof(opts));
     606          18 :         snprintf(opts.bstype.bstype, sizeof(opts.bstype.bstype), SPDK_BLOBFS_SIGNATURE);
     607          18 :         if (opt) {
     608           0 :                 opts.cluster_sz = opt->cluster_sz;
     609             :         }
     610          18 :         spdk_bs_init(dev, &opts, init_cb, req);
     611             : }
     612             : 
     613             : static struct spdk_file *
     614          18 : file_alloc(struct spdk_filesystem *fs)
     615             : {
     616             :         struct spdk_file *file;
     617             : 
     618          18 :         file = calloc(1, sizeof(*file));
     619          18 :         if (file == NULL) {
     620           0 :                 return NULL;
     621             :         }
     622             : 
     623          18 :         file->tree = calloc(1, sizeof(*file->tree));
     624          18 :         if (file->tree == NULL) {
     625           0 :                 free(file);
     626           0 :                 return NULL;
     627             :         }
     628             : 
     629          18 :         if (pthread_spin_init(&file->lock, 0)) {
     630           0 :                 free(file->tree);
     631           0 :                 free(file);
     632           0 :                 return NULL;
     633             :         }
     634             : 
     635          18 :         file->fs = fs;
     636          18 :         TAILQ_INIT(&file->open_requests);
     637          18 :         TAILQ_INIT(&file->sync_requests);
     638          18 :         TAILQ_INSERT_TAIL(&fs->files, file, tailq);
     639          18 :         file->priority = SPDK_FILE_PRIORITY_LOW;
     640          18 :         return file;
     641             : }
     642             : 
     643             : static void fs_load_done(void *ctx, int bserrno);
     644             : 
     645             : static int
     646           2 : _handle_deleted_files(struct spdk_fs_request *req)
     647             : {
     648           2 :         struct spdk_fs_cb_args *args = &req->args;
     649           2 :         struct spdk_filesystem *fs = args->fs;
     650             : 
     651           2 :         if (!TAILQ_EMPTY(&args->op.fs_load.deleted_files)) {
     652             :                 struct spdk_deleted_file *deleted_file;
     653             : 
     654           0 :                 deleted_file = TAILQ_FIRST(&args->op.fs_load.deleted_files);
     655           0 :                 TAILQ_REMOVE(&args->op.fs_load.deleted_files, deleted_file, tailq);
     656           0 :                 spdk_bs_delete_blob(fs->bs, deleted_file->id, fs_load_done, req);
     657           0 :                 free(deleted_file);
     658           0 :                 return 0;
     659             :         }
     660             : 
     661           2 :         return 1;
     662             : }
     663             : 
     664             : static void
     665           2 : fs_load_done(void *ctx, int bserrno)
     666             : {
     667           2 :         struct spdk_fs_request *req = ctx;
     668           2 :         struct spdk_fs_cb_args *args = &req->args;
     669           2 :         struct spdk_filesystem *fs = args->fs;
     670             : 
     671             :         /* The filesystem has been loaded.  Now check if there are any files that
     672             :          *  were marked for deletion before last unload.  Do not complete the
     673             :          *  fs_load callback until all of them have been deleted on disk.
     674             :          */
     675           2 :         if (_handle_deleted_files(req) == 0) {
     676             :                 /* We found a file that's been marked for deleting but not actually
     677             :                  *  deleted yet.  This function will get called again once the delete
     678             :                  *  operation is completed.
     679             :                  */
     680           0 :                 return;
     681             :         }
     682             : 
     683           2 :         args->fn.fs_op_with_handle(args->arg, fs, 0);
     684           2 :         free_fs_request(req);
     685             : 
     686             : }
     687             : 
     688             : static void
     689           2 : iter_cb(void *ctx, struct spdk_blob *blob, int rc)
     690             : {
     691           2 :         struct spdk_fs_request *req = ctx;
     692           2 :         struct spdk_fs_cb_args *args = &req->args;
     693           2 :         struct spdk_filesystem *fs = args->fs;
     694           2 :         uint64_t *length;
     695           2 :         const char *name;
     696           2 :         uint32_t *is_deleted;
     697           2 :         size_t value_len;
     698             : 
     699           2 :         if (rc < 0) {
     700           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     701           0 :                 free_fs_request(req);
     702           0 :                 return;
     703             :         }
     704             : 
     705           2 :         rc = spdk_blob_get_xattr_value(blob, "name", (const void **)&name, &value_len);
     706           2 :         if (rc < 0) {
     707           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     708           0 :                 free_fs_request(req);
     709           0 :                 return;
     710             :         }
     711             : 
     712           2 :         rc = spdk_blob_get_xattr_value(blob, "length", (const void **)&length, &value_len);
     713           2 :         if (rc < 0) {
     714           0 :                 args->fn.fs_op_with_handle(args->arg, fs, rc);
     715           0 :                 free_fs_request(req);
     716           0 :                 return;
     717             :         }
     718             : 
     719           2 :         assert(value_len == 8);
     720             : 
     721             :         /* This file could be deleted last time without close it, then app crashed, so we delete it now */
     722           2 :         rc = spdk_blob_get_xattr_value(blob, "is_deleted", (const void **)&is_deleted, &value_len);
     723           2 :         if (rc < 0) {
     724             :                 struct spdk_file *f;
     725             : 
     726           2 :                 f = file_alloc(fs);
     727           2 :                 if (f == NULL) {
     728           0 :                         SPDK_ERRLOG("Cannot allocate file to handle deleted file on disk\n");
     729           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     730           0 :                         free_fs_request(req);
     731           0 :                         return;
     732             :                 }
     733             : 
     734           2 :                 f->name = strdup(name);
     735           2 :                 if (!f->name) {
     736           0 :                         SPDK_ERRLOG("Cannot allocate memory for file name\n");
     737           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     738           0 :                         free_fs_request(req);
     739           0 :                         file_free(f);
     740           0 :                         return;
     741             :                 }
     742             : 
     743           2 :                 f->blobid = spdk_blob_get_id(blob);
     744           2 :                 f->length = *length;
     745           2 :                 f->length_flushed = *length;
     746           2 :                 f->length_xattr = *length;
     747           2 :                 f->append_pos = *length;
     748           2 :                 SPDK_DEBUGLOG(blobfs, "added file %s length=%ju\n", f->name, f->length);
     749             :         } else {
     750             :                 struct spdk_deleted_file *deleted_file;
     751             : 
     752           0 :                 deleted_file = calloc(1, sizeof(*deleted_file));
     753           0 :                 if (deleted_file == NULL) {
     754           0 :                         args->fn.fs_op_with_handle(args->arg, fs, -ENOMEM);
     755           0 :                         free_fs_request(req);
     756           0 :                         return;
     757             :                 }
     758           0 :                 deleted_file->id = spdk_blob_get_id(blob);
     759           0 :                 TAILQ_INSERT_TAIL(&args->op.fs_load.deleted_files, deleted_file, tailq);
     760             :         }
     761             : }
     762             : 
     763             : static void
     764           2 : load_cb(void *ctx, struct spdk_blob_store *bs, int bserrno)
     765             : {
     766           2 :         struct spdk_fs_request *req = ctx;
     767           2 :         struct spdk_fs_cb_args *args = &req->args;
     768           2 :         struct spdk_filesystem *fs = args->fs;
     769           2 :         struct spdk_bs_type bstype;
     770             :         static const struct spdk_bs_type blobfs_type = {SPDK_BLOBFS_SIGNATURE};
     771             :         static const struct spdk_bs_type zeros;
     772             : 
     773           2 :         if (bserrno != 0) {
     774           0 :                 args->fn.fs_op_with_handle(args->arg, NULL, bserrno);
     775           0 :                 free_fs_request(req);
     776           0 :                 fs_free_io_channels(fs);
     777           0 :                 fs_io_device_unregister(fs);
     778           0 :                 return;
     779             :         }
     780             : 
     781           2 :         bstype = spdk_bs_get_bstype(bs);
     782             : 
     783           2 :         if (!memcmp(&bstype, &zeros, sizeof(bstype))) {
     784           0 :                 SPDK_DEBUGLOG(blobfs, "assigning bstype\n");
     785           0 :                 spdk_bs_set_bstype(bs, blobfs_type);
     786           2 :         } else if (memcmp(&bstype, &blobfs_type, sizeof(bstype))) {
     787           0 :                 SPDK_ERRLOG("not blobfs\n");
     788           0 :                 SPDK_LOGDUMP(blobfs, "bstype", &bstype, sizeof(bstype));
     789           0 :                 args->fn.fs_op_with_handle(args->arg, NULL, -EINVAL);
     790           0 :                 free_fs_request(req);
     791           0 :                 fs_free_io_channels(fs);
     792           0 :                 fs_io_device_unregister(fs);
     793           0 :                 return;
     794             :         }
     795             : 
     796           2 :         common_fs_bs_init(fs, bs);
     797           2 :         fs_load_done(req, 0);
     798             : }
     799             : 
     800             : static void
     801          20 : fs_io_device_unregister(struct spdk_filesystem *fs)
     802             : {
     803          20 :         assert(fs != NULL);
     804          20 :         spdk_io_device_unregister(&fs->md_target, NULL);
     805          20 :         spdk_io_device_unregister(&fs->sync_target, NULL);
     806          20 :         spdk_io_device_unregister(&fs->io_target, NULL);
     807          20 :         free(fs);
     808          20 : }
     809             : 
     810             : static void
     811          20 : fs_free_io_channels(struct spdk_filesystem *fs)
     812             : {
     813          20 :         assert(fs != NULL);
     814          20 :         spdk_fs_free_io_channel(fs->md_target.md_io_channel);
     815          20 :         spdk_fs_free_io_channel(fs->sync_target.sync_io_channel);
     816          20 : }
     817             : 
     818             : void
     819           2 : spdk_fs_load(struct spdk_bs_dev *dev, fs_send_request_fn send_request_fn,
     820             :              spdk_fs_op_with_handle_complete cb_fn, void *cb_arg)
     821             : {
     822             :         struct spdk_filesystem *fs;
     823             :         struct spdk_fs_cb_args *args;
     824             :         struct spdk_fs_request *req;
     825           2 :         struct spdk_bs_opts     bs_opts;
     826             : 
     827           2 :         fs = fs_alloc(dev, send_request_fn);
     828           2 :         if (fs == NULL) {
     829           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     830           0 :                 return;
     831             :         }
     832             : 
     833           2 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
     834           2 :         if (req == NULL) {
     835           0 :                 fs_free_io_channels(fs);
     836           0 :                 fs_io_device_unregister(fs);
     837           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
     838           0 :                 return;
     839             :         }
     840             : 
     841           2 :         args = &req->args;
     842           2 :         args->fn.fs_op_with_handle = cb_fn;
     843           2 :         args->arg = cb_arg;
     844           2 :         args->fs = fs;
     845           2 :         TAILQ_INIT(&args->op.fs_load.deleted_files);
     846           2 :         spdk_bs_opts_init(&bs_opts, sizeof(bs_opts));
     847           2 :         bs_opts.iter_cb_fn = iter_cb;
     848           2 :         bs_opts.iter_cb_arg = req;
     849           2 :         spdk_bs_load(dev, &bs_opts, load_cb, req);
     850             : }
     851             : 
     852             : static void
     853          20 : unload_cb(void *ctx, int bserrno)
     854             : {
     855          20 :         struct spdk_fs_request *req = ctx;
     856          20 :         struct spdk_fs_cb_args *args = &req->args;
     857          20 :         struct spdk_filesystem *fs = args->fs;
     858             :         struct spdk_file *file, *tmp;
     859             : 
     860          26 :         TAILQ_FOREACH_SAFE(file, &fs->files, tailq, tmp) {
     861           6 :                 TAILQ_REMOVE(&fs->files, file, tailq);
     862           6 :                 file_free(file);
     863             :         }
     864             : 
     865          20 :         free_global_cache();
     866             : 
     867          20 :         args->fn.fs_op(args->arg, bserrno);
     868          20 :         free(req);
     869             : 
     870          20 :         fs_io_device_unregister(fs);
     871          20 : }
     872             : 
     873             : void
     874          20 : spdk_fs_unload(struct spdk_filesystem *fs, spdk_fs_op_complete cb_fn, void *cb_arg)
     875             : {
     876             :         struct spdk_fs_request *req;
     877             :         struct spdk_fs_cb_args *args;
     878             : 
     879             :         /*
     880             :          * We must free the md_channel before unloading the blobstore, so just
     881             :          *  allocate this request from the general heap.
     882             :          */
     883          20 :         req = calloc(1, sizeof(*req));
     884          20 :         if (req == NULL) {
     885           0 :                 cb_fn(cb_arg, -ENOMEM);
     886           0 :                 return;
     887             :         }
     888             : 
     889          20 :         args = &req->args;
     890          20 :         args->fn.fs_op = cb_fn;
     891          20 :         args->arg = cb_arg;
     892          20 :         args->fs = fs;
     893             : 
     894          20 :         fs_free_io_channels(fs);
     895          20 :         spdk_bs_unload(fs->bs, unload_cb, req);
     896             : }
     897             : 
     898             : static struct spdk_file *
     899          75 : fs_find_file(struct spdk_filesystem *fs, const char *name)
     900             : {
     901             :         struct spdk_file *file;
     902             : 
     903          82 :         TAILQ_FOREACH(file, &fs->files, tailq) {
     904          48 :                 if (!strncmp(name, file->name, SPDK_FILE_NAME_MAX)) {
     905          41 :                         return file;
     906             :                 }
     907             :         }
     908             : 
     909          34 :         return NULL;
     910             : }
     911             : 
     912             : void
     913           4 : spdk_fs_file_stat_async(struct spdk_filesystem *fs, const char *name,
     914             :                         spdk_file_stat_op_complete cb_fn, void *cb_arg)
     915             : {
     916           4 :         struct spdk_file_stat stat;
     917           4 :         struct spdk_file *f = NULL;
     918             : 
     919           4 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
     920           0 :                 cb_fn(cb_arg, NULL, -ENAMETOOLONG);
     921           0 :                 return;
     922             :         }
     923             : 
     924           4 :         f = fs_find_file(fs, name);
     925           4 :         if (f != NULL) {
     926           4 :                 stat.blobid = f->blobid;
     927           4 :                 stat.size = f->append_pos >= f->length ? f->append_pos : f->length;
     928           4 :                 cb_fn(cb_arg, &stat, 0);
     929           4 :                 return;
     930             :         }
     931             : 
     932           0 :         cb_fn(cb_arg, NULL, -ENOENT);
     933             : }
     934             : 
     935             : static void
     936           4 : __copy_stat(void *arg, struct spdk_file_stat *stat, int fserrno)
     937             : {
     938           4 :         struct spdk_fs_request *req = arg;
     939           4 :         struct spdk_fs_cb_args *args = &req->args;
     940             : 
     941           4 :         args->rc = fserrno;
     942           4 :         if (fserrno == 0) {
     943           4 :                 memcpy(args->arg, stat, sizeof(*stat));
     944             :         }
     945           4 :         sem_post(args->sem);
     946           4 : }
     947             : 
     948             : static void
     949           4 : __file_stat(void *arg)
     950             : {
     951           4 :         struct spdk_fs_request *req = arg;
     952           4 :         struct spdk_fs_cb_args *args = &req->args;
     953             : 
     954           4 :         spdk_fs_file_stat_async(args->fs, args->op.stat.name,
     955             :                                 args->fn.stat_op, req);
     956           4 : }
     957             : 
     958             : int
     959           4 : spdk_fs_file_stat(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
     960             :                   const char *name, struct spdk_file_stat *stat)
     961             : {
     962           4 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
     963             :         struct spdk_fs_request *req;
     964             :         int rc;
     965             : 
     966           4 :         req = alloc_fs_request(channel);
     967           4 :         if (req == NULL) {
     968           0 :                 SPDK_ERRLOG("Cannot allocate stat req on file=%s\n", name);
     969           0 :                 return -ENOMEM;
     970             :         }
     971             : 
     972           4 :         req->args.fs = fs;
     973           4 :         req->args.op.stat.name = name;
     974           4 :         req->args.fn.stat_op = __copy_stat;
     975           4 :         req->args.arg = stat;
     976           4 :         req->args.sem = &channel->sem;
     977           4 :         channel->send_request(__file_stat, req);
     978           4 :         sem_wait(&channel->sem);
     979             : 
     980           4 :         rc = req->args.rc;
     981           4 :         free_fs_request(req);
     982             : 
     983           4 :         return rc;
     984             : }
     985             : 
     986             : static void
     987          16 : fs_create_blob_close_cb(void *ctx, int bserrno)
     988             : {
     989             :         int rc;
     990          16 :         struct spdk_fs_request *req = ctx;
     991          16 :         struct spdk_fs_cb_args *args = &req->args;
     992             : 
     993          16 :         rc = args->rc ? args->rc : bserrno;
     994          16 :         args->fn.file_op(args->arg, rc);
     995          16 :         free_fs_request(req);
     996          16 : }
     997             : 
     998             : static void
     999          16 : fs_create_blob_resize_cb(void *ctx, int bserrno)
    1000             : {
    1001          16 :         struct spdk_fs_request *req = ctx;
    1002          16 :         struct spdk_fs_cb_args *args = &req->args;
    1003          16 :         struct spdk_file *f = args->file;
    1004          16 :         struct spdk_blob *blob = args->op.create.blob;
    1005          16 :         uint64_t length = 0;
    1006             : 
    1007          16 :         args->rc = bserrno;
    1008          16 :         if (bserrno) {
    1009           0 :                 spdk_blob_close(blob, fs_create_blob_close_cb, args);
    1010           0 :                 return;
    1011             :         }
    1012             : 
    1013          16 :         spdk_blob_set_xattr(blob, "name", f->name, strlen(f->name) + 1);
    1014          16 :         spdk_blob_set_xattr(blob, "length", &length, sizeof(length));
    1015             : 
    1016          16 :         spdk_blob_close(blob, fs_create_blob_close_cb, args);
    1017             : }
    1018             : 
    1019             : static void
    1020          16 : fs_create_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
    1021             : {
    1022          16 :         struct spdk_fs_request *req = ctx;
    1023          16 :         struct spdk_fs_cb_args *args = &req->args;
    1024             : 
    1025          16 :         if (bserrno) {
    1026           0 :                 args->fn.file_op(args->arg, bserrno);
    1027           0 :                 free_fs_request(req);
    1028           0 :                 return;
    1029             :         }
    1030             : 
    1031          16 :         args->op.create.blob = blob;
    1032          16 :         spdk_blob_resize(blob, 1, fs_create_blob_resize_cb, req);
    1033             : }
    1034             : 
    1035             : static void
    1036          16 : fs_create_blob_create_cb(void *ctx, spdk_blob_id blobid, int bserrno)
    1037             : {
    1038          16 :         struct spdk_fs_request *req = ctx;
    1039          16 :         struct spdk_fs_cb_args *args = &req->args;
    1040          16 :         struct spdk_file *f = args->file;
    1041             : 
    1042          16 :         if (bserrno) {
    1043           0 :                 args->fn.file_op(args->arg, bserrno);
    1044           0 :                 free_fs_request(req);
    1045           0 :                 return;
    1046             :         }
    1047             : 
    1048          16 :         f->blobid = blobid;
    1049          16 :         spdk_bs_open_blob(f->fs->bs, blobid, fs_create_blob_open_cb, req);
    1050             : }
    1051             : 
    1052             : void
    1053          19 : spdk_fs_create_file_async(struct spdk_filesystem *fs, const char *name,
    1054             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1055             : {
    1056             :         struct spdk_file *file;
    1057             :         struct spdk_fs_request *req;
    1058             :         struct spdk_fs_cb_args *args;
    1059             : 
    1060          19 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1061           1 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1062           1 :                 return;
    1063             :         }
    1064             : 
    1065          18 :         file = fs_find_file(fs, name);
    1066          18 :         if (file != NULL) {
    1067           2 :                 cb_fn(cb_arg, -EEXIST);
    1068           2 :                 return;
    1069             :         }
    1070             : 
    1071          16 :         file = file_alloc(fs);
    1072          16 :         if (file == NULL) {
    1073           0 :                 SPDK_ERRLOG("Cannot allocate new file for creation\n");
    1074           0 :                 cb_fn(cb_arg, -ENOMEM);
    1075           0 :                 return;
    1076             :         }
    1077             : 
    1078          16 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1079          16 :         if (req == NULL) {
    1080           0 :                 SPDK_ERRLOG("Cannot allocate create async req for file=%s\n", name);
    1081           0 :                 TAILQ_REMOVE(&fs->files, file, tailq);
    1082           0 :                 file_free(file);
    1083           0 :                 cb_fn(cb_arg, -ENOMEM);
    1084           0 :                 return;
    1085             :         }
    1086             : 
    1087          16 :         args = &req->args;
    1088          16 :         args->file = file;
    1089          16 :         args->fn.file_op = cb_fn;
    1090          16 :         args->arg = cb_arg;
    1091             : 
    1092          16 :         file->name = strdup(name);
    1093          16 :         if (!file->name) {
    1094           0 :                 SPDK_ERRLOG("Cannot allocate file->name for file=%s\n", name);
    1095           0 :                 free_fs_request(req);
    1096           0 :                 TAILQ_REMOVE(&fs->files, file, tailq);
    1097           0 :                 file_free(file);
    1098           0 :                 cb_fn(cb_arg, -ENOMEM);
    1099           0 :                 return;
    1100             :         }
    1101          16 :         spdk_bs_create_blob(fs->bs, fs_create_blob_create_cb, args);
    1102             : }
    1103             : 
    1104             : static void
    1105           2 : __fs_create_file_done(void *arg, int fserrno)
    1106             : {
    1107           2 :         struct spdk_fs_request *req = arg;
    1108           2 :         struct spdk_fs_cb_args *args = &req->args;
    1109             : 
    1110           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
    1111           2 :         __wake_caller(args, fserrno);
    1112           2 : }
    1113             : 
    1114             : static void
    1115           2 : __fs_create_file(void *arg)
    1116             : {
    1117           2 :         struct spdk_fs_request *req = arg;
    1118           2 :         struct spdk_fs_cb_args *args = &req->args;
    1119             : 
    1120           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.create.name);
    1121           2 :         spdk_fs_create_file_async(args->fs, args->op.create.name, __fs_create_file_done, req);
    1122           2 : }
    1123             : 
    1124             : int
    1125           2 : spdk_fs_create_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx, const char *name)
    1126             : {
    1127           2 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1128             :         struct spdk_fs_request *req;
    1129             :         struct spdk_fs_cb_args *args;
    1130             :         int rc;
    1131             : 
    1132           2 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1133             : 
    1134           2 :         req = alloc_fs_request(channel);
    1135           2 :         if (req == NULL) {
    1136           0 :                 SPDK_ERRLOG("Cannot allocate req to create file=%s\n", name);
    1137           0 :                 return -ENOMEM;
    1138             :         }
    1139             : 
    1140           2 :         args = &req->args;
    1141           2 :         args->fs = fs;
    1142           2 :         args->op.create.name = name;
    1143           2 :         args->sem = &channel->sem;
    1144           2 :         fs->send_request(__fs_create_file, req);
    1145           2 :         sem_wait(&channel->sem);
    1146           2 :         rc = args->rc;
    1147           2 :         free_fs_request(req);
    1148             : 
    1149           2 :         return rc;
    1150             : }
    1151             : 
    1152             : static void
    1153          17 : fs_open_blob_done(void *ctx, struct spdk_blob *blob, int bserrno)
    1154             : {
    1155          17 :         struct spdk_fs_request *req = ctx;
    1156          17 :         struct spdk_fs_cb_args *args = &req->args;
    1157          17 :         struct spdk_file *f = args->file;
    1158             : 
    1159          17 :         f->blob = blob;
    1160          34 :         while (!TAILQ_EMPTY(&f->open_requests)) {
    1161          17 :                 req = TAILQ_FIRST(&f->open_requests);
    1162          17 :                 args = &req->args;
    1163          17 :                 TAILQ_REMOVE(&f->open_requests, req, args.op.open.tailq);
    1164          17 :                 spdk_trace_record(TRACE_BLOBFS_OPEN, 0, 0, 0, f->name);
    1165          17 :                 args->fn.file_op_with_handle(args->arg, f, bserrno);
    1166          17 :                 free_fs_request(req);
    1167             :         }
    1168          17 : }
    1169             : 
    1170             : static void
    1171          17 : fs_open_blob_create_cb(void *ctx, int bserrno)
    1172             : {
    1173          17 :         struct spdk_fs_request *req = ctx;
    1174          17 :         struct spdk_fs_cb_args *args = &req->args;
    1175          17 :         struct spdk_file *file = args->file;
    1176          17 :         struct spdk_filesystem *fs = args->fs;
    1177             : 
    1178          17 :         if (file == NULL) {
    1179             :                 /*
    1180             :                  * This is from an open with CREATE flag - the file
    1181             :                  *  is now created so look it up in the file list for this
    1182             :                  *  filesystem.
    1183             :                  */
    1184          13 :                 file = fs_find_file(fs, args->op.open.name);
    1185          13 :                 assert(file != NULL);
    1186          13 :                 args->file = file;
    1187             :         }
    1188             : 
    1189          17 :         file->ref_count++;
    1190          17 :         TAILQ_INSERT_TAIL(&file->open_requests, req, args.op.open.tailq);
    1191          17 :         if (file->ref_count == 1) {
    1192          17 :                 assert(file->blob == NULL);
    1193          17 :                 spdk_bs_open_blob(fs->bs, file->blobid, fs_open_blob_done, req);
    1194           0 :         } else if (file->blob != NULL) {
    1195           0 :                 fs_open_blob_done(req, file->blob, 0);
    1196             :         } else {
    1197             :                 /*
    1198             :                  * The blob open for this file is in progress due to a previous
    1199             :                  *  open request.  When that open completes, it will invoke the
    1200             :                  *  open callback for this request.
    1201             :                  */
    1202             :         }
    1203          17 : }
    1204             : 
    1205             : void
    1206          21 : spdk_fs_open_file_async(struct spdk_filesystem *fs, const char *name, uint32_t flags,
    1207             :                         spdk_file_op_with_handle_complete cb_fn, void *cb_arg)
    1208             : {
    1209          21 :         struct spdk_file *f = NULL;
    1210             :         struct spdk_fs_request *req;
    1211             :         struct spdk_fs_cb_args *args;
    1212             : 
    1213          21 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1214           1 :                 cb_fn(cb_arg, NULL, -ENAMETOOLONG);
    1215           1 :                 return;
    1216             :         }
    1217             : 
    1218          20 :         f = fs_find_file(fs, name);
    1219          20 :         if (f == NULL && !(flags & SPDK_BLOBFS_OPEN_CREATE)) {
    1220           2 :                 cb_fn(cb_arg, NULL, -ENOENT);
    1221           2 :                 return;
    1222             :         }
    1223             : 
    1224          18 :         if (f != NULL && f->is_deleted == true) {
    1225           1 :                 cb_fn(cb_arg, NULL, -ENOENT);
    1226           1 :                 return;
    1227             :         }
    1228             : 
    1229          17 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1230          17 :         if (req == NULL) {
    1231           0 :                 SPDK_ERRLOG("Cannot allocate async open req for file=%s\n", name);
    1232           0 :                 cb_fn(cb_arg, NULL, -ENOMEM);
    1233           0 :                 return;
    1234             :         }
    1235             : 
    1236          17 :         args = &req->args;
    1237          17 :         args->fn.file_op_with_handle = cb_fn;
    1238          17 :         args->arg = cb_arg;
    1239          17 :         args->file = f;
    1240          17 :         args->fs = fs;
    1241          17 :         args->op.open.name = name;
    1242             : 
    1243          17 :         if (f == NULL) {
    1244          13 :                 spdk_fs_create_file_async(fs, name, fs_open_blob_create_cb, req);
    1245             :         } else {
    1246           4 :                 fs_open_blob_create_cb(req, 0);
    1247             :         }
    1248             : }
    1249             : 
    1250             : static void
    1251          13 : __fs_open_file_done(void *arg, struct spdk_file *file, int bserrno)
    1252             : {
    1253          13 :         struct spdk_fs_request *req = arg;
    1254          13 :         struct spdk_fs_cb_args *args = &req->args;
    1255             : 
    1256          13 :         args->file = file;
    1257          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
    1258          13 :         __wake_caller(args, bserrno);
    1259          13 : }
    1260             : 
    1261             : static void
    1262          13 : __fs_open_file(void *arg)
    1263             : {
    1264          13 :         struct spdk_fs_request *req = arg;
    1265          13 :         struct spdk_fs_cb_args *args = &req->args;
    1266             : 
    1267          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", args->op.open.name);
    1268          13 :         spdk_fs_open_file_async(args->fs, args->op.open.name, args->op.open.flags,
    1269             :                                 __fs_open_file_done, req);
    1270          13 : }
    1271             : 
    1272             : int
    1273          13 : spdk_fs_open_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1274             :                   const char *name, uint32_t flags, struct spdk_file **file)
    1275             : {
    1276          13 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1277             :         struct spdk_fs_request *req;
    1278             :         struct spdk_fs_cb_args *args;
    1279             :         int rc;
    1280             : 
    1281          13 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1282             : 
    1283          13 :         req = alloc_fs_request(channel);
    1284          13 :         if (req == NULL) {
    1285           0 :                 SPDK_ERRLOG("Cannot allocate req for opening file=%s\n", name);
    1286           0 :                 return -ENOMEM;
    1287             :         }
    1288             : 
    1289          13 :         args = &req->args;
    1290          13 :         args->fs = fs;
    1291          13 :         args->op.open.name = name;
    1292          13 :         args->op.open.flags = flags;
    1293          13 :         args->sem = &channel->sem;
    1294          13 :         fs->send_request(__fs_open_file, req);
    1295          13 :         sem_wait(&channel->sem);
    1296          13 :         rc = args->rc;
    1297          13 :         if (rc == 0) {
    1298          11 :                 *file = args->file;
    1299             :         } else {
    1300           2 :                 *file = NULL;
    1301             :         }
    1302          13 :         free_fs_request(req);
    1303             : 
    1304          13 :         return rc;
    1305             : }
    1306             : 
    1307             : static void
    1308           2 : fs_rename_blob_close_cb(void *ctx, int bserrno)
    1309             : {
    1310           2 :         struct spdk_fs_request *req = ctx;
    1311           2 :         struct spdk_fs_cb_args *args = &req->args;
    1312             : 
    1313           2 :         args->fn.fs_op(args->arg, bserrno);
    1314           2 :         free_fs_request(req);
    1315           2 : }
    1316             : 
    1317             : static void
    1318           2 : fs_rename_blob_open_cb(void *ctx, struct spdk_blob *blob, int bserrno)
    1319             : {
    1320           2 :         struct spdk_fs_request *req = ctx;
    1321           2 :         struct spdk_fs_cb_args *args = &req->args;
    1322           2 :         const char *new_name = args->op.rename.new_name;
    1323             : 
    1324           2 :         spdk_blob_set_xattr(blob, "name", new_name, strlen(new_name) + 1);
    1325           2 :         spdk_blob_close(blob, fs_rename_blob_close_cb, req);
    1326           2 : }
    1327             : 
    1328             : static void
    1329           2 : _fs_md_rename_file(struct spdk_fs_request *req)
    1330             : {
    1331           2 :         struct spdk_fs_cb_args *args = &req->args;
    1332             :         struct spdk_file *f;
    1333             : 
    1334           2 :         f = fs_find_file(args->fs, args->op.rename.old_name);
    1335           2 :         if (f == NULL) {
    1336           0 :                 args->fn.fs_op(args->arg, -ENOENT);
    1337           0 :                 free_fs_request(req);
    1338           0 :                 return;
    1339             :         }
    1340             : 
    1341           2 :         free(f->name);
    1342           2 :         f->name = strdup(args->op.rename.new_name);
    1343           2 :         if (!f->name) {
    1344           0 :                 SPDK_ERRLOG("Cannot allocate memory for file name\n");
    1345           0 :                 args->fn.fs_op(args->arg, -ENOMEM);
    1346           0 :                 free_fs_request(req);
    1347           0 :                 return;
    1348             :         }
    1349             : 
    1350           2 :         args->file = f;
    1351           2 :         spdk_bs_open_blob(args->fs->bs, f->blobid, fs_rename_blob_open_cb, req);
    1352             : }
    1353             : 
    1354             : static void
    1355           1 : fs_rename_delete_done(void *arg, int fserrno)
    1356             : {
    1357           1 :         _fs_md_rename_file(arg);
    1358           1 : }
    1359             : 
    1360             : void
    1361           2 : spdk_fs_rename_file_async(struct spdk_filesystem *fs,
    1362             :                           const char *old_name, const char *new_name,
    1363             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1364             : {
    1365             :         struct spdk_file *f;
    1366             :         struct spdk_fs_request *req;
    1367             :         struct spdk_fs_cb_args *args;
    1368             : 
    1369           2 :         SPDK_DEBUGLOG(blobfs, "old=%s new=%s\n", old_name, new_name);
    1370           2 :         if (strnlen(new_name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1371           0 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1372           0 :                 return;
    1373             :         }
    1374             : 
    1375           2 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1376           2 :         if (req == NULL) {
    1377           0 :                 SPDK_ERRLOG("Cannot allocate rename async req for renaming file from %s to %s\n", old_name,
    1378             :                             new_name);
    1379           0 :                 cb_fn(cb_arg, -ENOMEM);
    1380           0 :                 return;
    1381             :         }
    1382             : 
    1383           2 :         args = &req->args;
    1384           2 :         args->fn.fs_op = cb_fn;
    1385           2 :         args->fs = fs;
    1386           2 :         args->arg = cb_arg;
    1387           2 :         args->op.rename.old_name = old_name;
    1388           2 :         args->op.rename.new_name = new_name;
    1389             : 
    1390           2 :         f = fs_find_file(fs, new_name);
    1391           2 :         if (f == NULL) {
    1392           1 :                 _fs_md_rename_file(req);
    1393           1 :                 return;
    1394             :         }
    1395             : 
    1396             :         /*
    1397             :          * The rename overwrites an existing file.  So delete the existing file, then
    1398             :          *  do the actual rename.
    1399             :          */
    1400           1 :         spdk_fs_delete_file_async(fs, new_name, fs_rename_delete_done, req);
    1401             : }
    1402             : 
    1403             : static void
    1404           1 : __fs_rename_file_done(void *arg, int fserrno)
    1405             : {
    1406           1 :         struct spdk_fs_request *req = arg;
    1407           1 :         struct spdk_fs_cb_args *args = &req->args;
    1408             : 
    1409           1 :         __wake_caller(args, fserrno);
    1410           1 : }
    1411             : 
    1412             : static void
    1413           1 : __fs_rename_file(void *arg)
    1414             : {
    1415           1 :         struct spdk_fs_request *req = arg;
    1416           1 :         struct spdk_fs_cb_args *args = &req->args;
    1417             : 
    1418           1 :         spdk_fs_rename_file_async(args->fs, args->op.rename.old_name, args->op.rename.new_name,
    1419             :                                   __fs_rename_file_done, req);
    1420           1 : }
    1421             : 
    1422             : int
    1423           1 : spdk_fs_rename_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1424             :                     const char *old_name, const char *new_name)
    1425             : {
    1426           1 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1427             :         struct spdk_fs_request *req;
    1428             :         struct spdk_fs_cb_args *args;
    1429             :         int rc;
    1430             : 
    1431           1 :         req = alloc_fs_request(channel);
    1432           1 :         if (req == NULL) {
    1433           0 :                 SPDK_ERRLOG("Cannot allocate rename req for file=%s\n", old_name);
    1434           0 :                 return -ENOMEM;
    1435             :         }
    1436             : 
    1437           1 :         args = &req->args;
    1438             : 
    1439           1 :         args->fs = fs;
    1440           1 :         args->op.rename.old_name = old_name;
    1441           1 :         args->op.rename.new_name = new_name;
    1442           1 :         args->sem = &channel->sem;
    1443           1 :         fs->send_request(__fs_rename_file, req);
    1444           1 :         sem_wait(&channel->sem);
    1445           1 :         rc = args->rc;
    1446           1 :         free_fs_request(req);
    1447           1 :         return rc;
    1448             : }
    1449             : 
    1450             : static void
    1451          16 : blob_delete_cb(void *ctx, int bserrno)
    1452             : {
    1453          16 :         struct spdk_fs_request *req = ctx;
    1454          16 :         struct spdk_fs_cb_args *args = &req->args;
    1455             : 
    1456          16 :         args->fn.file_op(args->arg, bserrno);
    1457          16 :         free_fs_request(req);
    1458          16 : }
    1459             : 
    1460             : void
    1461          16 : spdk_fs_delete_file_async(struct spdk_filesystem *fs, const char *name,
    1462             :                           spdk_file_op_complete cb_fn, void *cb_arg)
    1463             : {
    1464             :         struct spdk_file *f;
    1465             :         spdk_blob_id blobid;
    1466             :         struct spdk_fs_request *req;
    1467             :         struct spdk_fs_cb_args *args;
    1468             : 
    1469          16 :         SPDK_DEBUGLOG(blobfs, "file=%s\n", name);
    1470             : 
    1471          16 :         if (strnlen(name, SPDK_FILE_NAME_MAX + 1) == SPDK_FILE_NAME_MAX + 1) {
    1472           0 :                 cb_fn(cb_arg, -ENAMETOOLONG);
    1473           0 :                 return;
    1474             :         }
    1475             : 
    1476          16 :         f = fs_find_file(fs, name);
    1477          16 :         if (f == NULL) {
    1478           2 :                 SPDK_ERRLOG("Cannot find the file=%s to deleted\n", name);
    1479           2 :                 cb_fn(cb_arg, -ENOENT);
    1480           2 :                 return;
    1481             :         }
    1482             : 
    1483          14 :         req = alloc_fs_request(fs->md_target.md_fs_channel);
    1484          14 :         if (req == NULL) {
    1485           0 :                 SPDK_ERRLOG("Cannot allocate the req for the file=%s to deleted\n", name);
    1486           0 :                 cb_fn(cb_arg, -ENOMEM);
    1487           0 :                 return;
    1488             :         }
    1489             : 
    1490          14 :         args = &req->args;
    1491          14 :         args->fn.file_op = cb_fn;
    1492          14 :         args->arg = cb_arg;
    1493             : 
    1494          14 :         if (f->ref_count > 0) {
    1495             :                 /* If the ref > 0, we mark the file as deleted and delete it when we close it. */
    1496           2 :                 f->is_deleted = true;
    1497           2 :                 spdk_blob_set_xattr(f->blob, "is_deleted", &f->is_deleted, sizeof(bool));
    1498           2 :                 spdk_blob_sync_md(f->blob, blob_delete_cb, req);
    1499           2 :                 return;
    1500             :         }
    1501             : 
    1502          12 :         blobid = f->blobid;
    1503          12 :         TAILQ_REMOVE(&fs->files, f, tailq);
    1504             : 
    1505          12 :         file_free(f);
    1506             : 
    1507          12 :         spdk_bs_delete_blob(fs->bs, blobid, blob_delete_cb, req);
    1508             : }
    1509             : 
    1510             : static void
    1511           8 : __fs_delete_file_done(void *arg, int fserrno)
    1512             : {
    1513           8 :         struct spdk_fs_request *req = arg;
    1514           8 :         struct spdk_fs_cb_args *args = &req->args;
    1515             : 
    1516           8 :         spdk_trace_record(TRACE_BLOBFS_DELETE_DONE, 0, 0, 0, args->op.delete.name);
    1517           8 :         __wake_caller(args, fserrno);
    1518           8 : }
    1519             : 
    1520             : static void
    1521           8 : __fs_delete_file(void *arg)
    1522             : {
    1523           8 :         struct spdk_fs_request *req = arg;
    1524           8 :         struct spdk_fs_cb_args *args = &req->args;
    1525             : 
    1526           8 :         spdk_trace_record(TRACE_BLOBFS_DELETE_START, 0, 0, 0, args->op.delete.name);
    1527           8 :         spdk_fs_delete_file_async(args->fs, args->op.delete.name, __fs_delete_file_done, req);
    1528           8 : }
    1529             : 
    1530             : int
    1531           8 : spdk_fs_delete_file(struct spdk_filesystem *fs, struct spdk_fs_thread_ctx *ctx,
    1532             :                     const char *name)
    1533             : {
    1534           8 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1535             :         struct spdk_fs_request *req;
    1536             :         struct spdk_fs_cb_args *args;
    1537             :         int rc;
    1538             : 
    1539           8 :         req = alloc_fs_request(channel);
    1540           8 :         if (req == NULL) {
    1541           0 :                 SPDK_DEBUGLOG(blobfs, "Cannot allocate req to delete file=%s\n", name);
    1542           0 :                 return -ENOMEM;
    1543             :         }
    1544             : 
    1545           8 :         args = &req->args;
    1546           8 :         args->fs = fs;
    1547           8 :         args->op.delete.name = name;
    1548           8 :         args->sem = &channel->sem;
    1549           8 :         fs->send_request(__fs_delete_file, req);
    1550           8 :         sem_wait(&channel->sem);
    1551           8 :         rc = args->rc;
    1552           8 :         free_fs_request(req);
    1553             : 
    1554           8 :         return rc;
    1555             : }
    1556             : 
    1557             : spdk_fs_iter
    1558           1 : spdk_fs_iter_first(struct spdk_filesystem *fs)
    1559             : {
    1560             :         struct spdk_file *f;
    1561             : 
    1562           1 :         f = TAILQ_FIRST(&fs->files);
    1563           1 :         return f;
    1564             : }
    1565             : 
    1566             : spdk_fs_iter
    1567           1 : spdk_fs_iter_next(spdk_fs_iter iter)
    1568             : {
    1569           1 :         struct spdk_file *f = iter;
    1570             : 
    1571           1 :         if (f == NULL) {
    1572           0 :                 return NULL;
    1573             :         }
    1574             : 
    1575           1 :         f = TAILQ_NEXT(f, tailq);
    1576           1 :         return f;
    1577             : }
    1578             : 
    1579             : const char *
    1580           2 : spdk_file_get_name(struct spdk_file *file)
    1581             : {
    1582           2 :         return file->name;
    1583             : }
    1584             : 
    1585             : uint64_t
    1586           6 : spdk_file_get_length(struct spdk_file *file)
    1587             : {
    1588             :         uint64_t length;
    1589             : 
    1590           6 :         assert(file != NULL);
    1591             : 
    1592           6 :         length = file->append_pos >= file->length ? file->append_pos : file->length;
    1593           6 :         SPDK_DEBUGLOG(blobfs, "file=%s length=0x%jx\n", file->name, length);
    1594           6 :         return length;
    1595             : }
    1596             : 
    1597             : static void
    1598           8 : fs_truncate_complete_cb(void *ctx, int bserrno)
    1599             : {
    1600           8 :         struct spdk_fs_request *req = ctx;
    1601           8 :         struct spdk_fs_cb_args *args = &req->args;
    1602             : 
    1603           8 :         args->fn.file_op(args->arg, bserrno);
    1604           8 :         free_fs_request(req);
    1605           8 : }
    1606             : 
    1607             : static void
    1608           8 : fs_truncate_resize_cb(void *ctx, int bserrno)
    1609             : {
    1610           8 :         struct spdk_fs_request *req = ctx;
    1611           8 :         struct spdk_fs_cb_args *args = &req->args;
    1612           8 :         struct spdk_file *file = args->file;
    1613           8 :         uint64_t *length = &args->op.truncate.length;
    1614             : 
    1615           8 :         if (bserrno) {
    1616           0 :                 args->fn.file_op(args->arg, bserrno);
    1617           0 :                 free_fs_request(req);
    1618           0 :                 return;
    1619             :         }
    1620             : 
    1621           8 :         spdk_blob_set_xattr(file->blob, "length", length, sizeof(*length));
    1622             : 
    1623           8 :         file->length = *length;
    1624           8 :         if (file->append_pos > file->length) {
    1625           0 :                 file->append_pos = file->length;
    1626             :         }
    1627             : 
    1628           8 :         spdk_blob_sync_md(file->blob, fs_truncate_complete_cb, req);
    1629             : }
    1630             : 
    1631             : static uint64_t
    1632           8 : __bytes_to_clusters(uint64_t length, uint64_t cluster_sz)
    1633             : {
    1634           8 :         return (length + cluster_sz - 1) / cluster_sz;
    1635             : }
    1636             : 
    1637             : void
    1638           9 : spdk_file_truncate_async(struct spdk_file *file, uint64_t length,
    1639             :                          spdk_file_op_complete cb_fn, void *cb_arg)
    1640             : {
    1641             :         struct spdk_filesystem *fs;
    1642             :         size_t num_clusters;
    1643             :         struct spdk_fs_request *req;
    1644             :         struct spdk_fs_cb_args *args;
    1645             : 
    1646           9 :         SPDK_DEBUGLOG(blobfs, "file=%s old=0x%jx new=0x%jx\n", file->name, file->length, length);
    1647           9 :         if (length == file->length) {
    1648           1 :                 cb_fn(cb_arg, 0);
    1649           1 :                 return;
    1650             :         }
    1651             : 
    1652           8 :         req = alloc_fs_request(file->fs->md_target.md_fs_channel);
    1653           8 :         if (req == NULL) {
    1654           0 :                 cb_fn(cb_arg, -ENOMEM);
    1655           0 :                 return;
    1656             :         }
    1657             : 
    1658           8 :         args = &req->args;
    1659           8 :         args->fn.file_op = cb_fn;
    1660           8 :         args->arg = cb_arg;
    1661           8 :         args->file = file;
    1662           8 :         args->op.truncate.length = length;
    1663           8 :         fs = file->fs;
    1664             : 
    1665           8 :         num_clusters = __bytes_to_clusters(length, fs->bs_opts.cluster_sz);
    1666             : 
    1667           8 :         spdk_blob_resize(file->blob, num_clusters, fs_truncate_resize_cb, req);
    1668             : }
    1669             : 
    1670             : static void
    1671           3 : __truncate(void *arg)
    1672             : {
    1673           3 :         struct spdk_fs_request *req = arg;
    1674           3 :         struct spdk_fs_cb_args *args = &req->args;
    1675             : 
    1676           3 :         spdk_file_truncate_async(args->file, args->op.truncate.length,
    1677             :                                  args->fn.file_op, args);
    1678           3 : }
    1679             : 
    1680             : int
    1681           3 : spdk_file_truncate(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    1682             :                    uint64_t length)
    1683             : {
    1684           3 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    1685             :         struct spdk_fs_request *req;
    1686             :         struct spdk_fs_cb_args *args;
    1687             :         int rc;
    1688             : 
    1689           3 :         req = alloc_fs_request(channel);
    1690           3 :         if (req == NULL) {
    1691           0 :                 return -ENOMEM;
    1692             :         }
    1693             : 
    1694           3 :         args = &req->args;
    1695             : 
    1696           3 :         args->file = file;
    1697           3 :         args->op.truncate.length = length;
    1698           3 :         args->fn.file_op = __wake_caller;
    1699           3 :         args->sem = &channel->sem;
    1700             : 
    1701           3 :         channel->send_request(__truncate, req);
    1702           3 :         sem_wait(&channel->sem);
    1703           3 :         rc = args->rc;
    1704           3 :         free_fs_request(req);
    1705             : 
    1706           3 :         return rc;
    1707             : }
    1708             : 
    1709             : static void
    1710           7 : __rw_done(void *ctx, int bserrno)
    1711             : {
    1712           7 :         struct spdk_fs_request *req = ctx;
    1713           7 :         struct spdk_fs_cb_args *args = &req->args;
    1714             : 
    1715           7 :         spdk_free(args->op.rw.pin_buf);
    1716           7 :         args->fn.file_op(args->arg, bserrno);
    1717           7 :         free_fs_request(req);
    1718           7 : }
    1719             : 
    1720             : static void
    1721           6 : __read_done(void *ctx, int bserrno)
    1722             : {
    1723           6 :         struct spdk_fs_request *req = ctx;
    1724           6 :         struct spdk_fs_cb_args *args = &req->args;
    1725             :         void *buf;
    1726             : 
    1727           6 :         if (bserrno) {
    1728           0 :                 __rw_done(req, bserrno);
    1729           0 :                 return;
    1730             :         }
    1731             : 
    1732           6 :         assert(req != NULL);
    1733           6 :         buf = (void *)((uintptr_t)args->op.rw.pin_buf + (args->op.rw.offset & (args->op.rw.blocklen - 1)));
    1734           6 :         if (args->op.rw.is_read) {
    1735           3 :                 spdk_copy_buf_to_iovs(args->iovs, args->iovcnt, buf, args->op.rw.length);
    1736           3 :                 __rw_done(req, 0);
    1737             :         } else {
    1738           3 :                 spdk_copy_iovs_to_buf(buf, args->op.rw.length, args->iovs, args->iovcnt);
    1739           3 :                 spdk_blob_io_write(args->file->blob, args->op.rw.channel,
    1740             :                                    args->op.rw.pin_buf,
    1741             :                                    args->op.rw.start_lba, args->op.rw.num_lba,
    1742             :                                    __rw_done, req);
    1743             :         }
    1744             : }
    1745             : 
    1746             : static void
    1747           6 : __do_blob_read(void *ctx, int fserrno)
    1748             : {
    1749           6 :         struct spdk_fs_request *req = ctx;
    1750           6 :         struct spdk_fs_cb_args *args = &req->args;
    1751             : 
    1752           6 :         if (fserrno) {
    1753           0 :                 __rw_done(req, fserrno);
    1754           0 :                 return;
    1755             :         }
    1756           6 :         spdk_blob_io_read(args->file->blob, args->op.rw.channel,
    1757             :                           args->op.rw.pin_buf,
    1758             :                           args->op.rw.start_lba, args->op.rw.num_lba,
    1759             :                           __read_done, req);
    1760             : }
    1761             : 
    1762             : static void
    1763          17 : __get_page_parameters(struct spdk_file *file, uint64_t offset, uint64_t length,
    1764             :                       uint64_t *start_lba, uint32_t *lba_size, uint64_t *num_lba)
    1765             : {
    1766             :         uint64_t end_lba;
    1767             : 
    1768          17 :         *lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
    1769          17 :         *start_lba = offset / *lba_size;
    1770          17 :         end_lba = (offset + length - 1) / *lba_size;
    1771          17 :         *num_lba = (end_lba - *start_lba + 1);
    1772          17 : }
    1773             : 
    1774             : static bool
    1775           1 : __is_lba_aligned(struct spdk_file *file, uint64_t offset, uint64_t length)
    1776             : {
    1777           1 :         uint32_t lba_size = spdk_bs_get_io_unit_size(file->fs->bs);
    1778             : 
    1779           1 :         if ((offset % lba_size == 0) && (length % lba_size == 0)) {
    1780           1 :                 return true;
    1781             :         }
    1782             : 
    1783           0 :         return false;
    1784             : }
    1785             : 
    1786             : static void
    1787           7 : _fs_request_setup_iovs(struct spdk_fs_request *req, struct iovec *iovs, uint32_t iovcnt)
    1788             : {
    1789             :         uint32_t i;
    1790             : 
    1791          18 :         for (i = 0; i < iovcnt; i++) {
    1792          11 :                 req->args.iovs[i].iov_base = iovs[i].iov_base;
    1793          11 :                 req->args.iovs[i].iov_len = iovs[i].iov_len;
    1794             :         }
    1795           7 : }
    1796             : 
    1797             : static void
    1798           7 : __readvwritev(struct spdk_file *file, struct spdk_io_channel *_channel,
    1799             :               struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1800             :               spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
    1801             : {
    1802             :         struct spdk_fs_request *req;
    1803             :         struct spdk_fs_cb_args *args;
    1804           7 :         struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
    1805           7 :         uint64_t start_lba, num_lba, pin_buf_length;
    1806           7 :         uint32_t lba_size;
    1807             : 
    1808           7 :         if (is_read && offset + length > file->length) {
    1809           0 :                 cb_fn(cb_arg, -EINVAL);
    1810           0 :                 return;
    1811             :         }
    1812             : 
    1813           7 :         req = alloc_fs_request_with_iov(channel, iovcnt);
    1814           7 :         if (req == NULL) {
    1815           0 :                 cb_fn(cb_arg, -ENOMEM);
    1816           0 :                 return;
    1817             :         }
    1818             : 
    1819           7 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    1820             : 
    1821           7 :         args = &req->args;
    1822           7 :         args->fn.file_op = cb_fn;
    1823           7 :         args->arg = cb_arg;
    1824           7 :         args->file = file;
    1825           7 :         args->op.rw.channel = channel->bs_channel;
    1826           7 :         _fs_request_setup_iovs(req, iovs, iovcnt);
    1827           7 :         args->op.rw.is_read = is_read;
    1828           7 :         args->op.rw.offset = offset;
    1829           7 :         args->op.rw.blocklen = lba_size;
    1830             : 
    1831           7 :         pin_buf_length = num_lba * lba_size;
    1832           7 :         args->op.rw.length = pin_buf_length;
    1833           7 :         args->op.rw.pin_buf = spdk_malloc(pin_buf_length, lba_size, NULL,
    1834             :                                           SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
    1835           7 :         if (args->op.rw.pin_buf == NULL) {
    1836           0 :                 SPDK_DEBUGLOG(blobfs, "Failed to allocate buf for: file=%s offset=%jx length=%jx\n",
    1837             :                               file->name, offset, length);
    1838           0 :                 free_fs_request(req);
    1839           0 :                 cb_fn(cb_arg, -ENOMEM);
    1840           0 :                 return;
    1841             :         }
    1842             : 
    1843           7 :         args->op.rw.start_lba = start_lba;
    1844           7 :         args->op.rw.num_lba = num_lba;
    1845             : 
    1846           7 :         if (!is_read && file->length < offset + length) {
    1847           3 :                 spdk_file_truncate_async(file, offset + length, __do_blob_read, req);
    1848           4 :         } else if (!is_read && __is_lba_aligned(file, offset, length)) {
    1849           1 :                 spdk_copy_iovs_to_buf(args->op.rw.pin_buf, args->op.rw.length, args->iovs, args->iovcnt);
    1850           1 :                 spdk_blob_io_write(args->file->blob, args->op.rw.channel,
    1851             :                                    args->op.rw.pin_buf,
    1852             :                                    args->op.rw.start_lba, args->op.rw.num_lba,
    1853             :                                    __rw_done, req);
    1854             :         } else {
    1855           3 :                 __do_blob_read(req, 0);
    1856             :         }
    1857             : }
    1858             : 
    1859             : static void
    1860           3 : __readwrite(struct spdk_file *file, struct spdk_io_channel *channel,
    1861             :             void *payload, uint64_t offset, uint64_t length,
    1862             :             spdk_file_op_complete cb_fn, void *cb_arg, int is_read)
    1863             : {
    1864           3 :         struct iovec iov;
    1865             : 
    1866           3 :         iov.iov_base = payload;
    1867           3 :         iov.iov_len = (size_t)length;
    1868             : 
    1869           3 :         __readvwritev(file, channel, &iov, 1, offset, length, cb_fn, cb_arg, is_read);
    1870           3 : }
    1871             : 
    1872             : void
    1873           2 : spdk_file_write_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1874             :                       void *payload, uint64_t offset, uint64_t length,
    1875             :                       spdk_file_op_complete cb_fn, void *cb_arg)
    1876             : {
    1877           2 :         __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 0);
    1878           2 : }
    1879             : 
    1880             : void
    1881           2 : spdk_file_writev_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1882             :                        struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1883             :                        spdk_file_op_complete cb_fn, void *cb_arg)
    1884             : {
    1885           2 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1886             :                       file->name, offset, length);
    1887             : 
    1888           2 :         __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 0);
    1889           2 : }
    1890             : 
    1891             : void
    1892           1 : spdk_file_read_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1893             :                      void *payload, uint64_t offset, uint64_t length,
    1894             :                      spdk_file_op_complete cb_fn, void *cb_arg)
    1895             : {
    1896           1 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1897             :                       file->name, offset, length);
    1898             : 
    1899           1 :         __readwrite(file, channel, payload, offset, length, cb_fn, cb_arg, 1);
    1900           1 : }
    1901             : 
    1902             : void
    1903           2 : spdk_file_readv_async(struct spdk_file *file, struct spdk_io_channel *channel,
    1904             :                       struct iovec *iovs, uint32_t iovcnt, uint64_t offset, uint64_t length,
    1905             :                       spdk_file_op_complete cb_fn, void *cb_arg)
    1906             : {
    1907           2 :         SPDK_DEBUGLOG(blobfs, "file=%s offset=%jx length=%jx\n",
    1908             :                       file->name, offset, length);
    1909             : 
    1910           2 :         __readvwritev(file, channel, iovs, iovcnt, offset, length, cb_fn, cb_arg, 1);
    1911           2 : }
    1912             : 
    1913             : struct spdk_io_channel *
    1914           1 : spdk_fs_alloc_io_channel(struct spdk_filesystem *fs)
    1915             : {
    1916             :         struct spdk_io_channel *io_channel;
    1917             :         struct spdk_fs_channel *fs_channel;
    1918             : 
    1919           1 :         io_channel = spdk_get_io_channel(&fs->io_target);
    1920           1 :         fs_channel = spdk_io_channel_get_ctx(io_channel);
    1921           1 :         fs_channel->bs_channel = spdk_bs_alloc_io_channel(fs->bs);
    1922           1 :         fs_channel->send_request = __send_request_direct;
    1923             : 
    1924           1 :         return io_channel;
    1925             : }
    1926             : 
    1927             : void
    1928          41 : spdk_fs_free_io_channel(struct spdk_io_channel *channel)
    1929             : {
    1930          41 :         spdk_put_io_channel(channel);
    1931          41 : }
    1932             : 
    1933             : struct spdk_fs_thread_ctx *
    1934          12 : spdk_fs_alloc_thread_ctx(struct spdk_filesystem *fs)
    1935             : {
    1936             :         struct spdk_fs_thread_ctx *ctx;
    1937             : 
    1938          12 :         ctx = calloc(1, sizeof(*ctx));
    1939          12 :         if (!ctx) {
    1940           0 :                 return NULL;
    1941             :         }
    1942             : 
    1943          12 :         if (pthread_spin_init(&ctx->ch.lock, 0)) {
    1944           0 :                 free(ctx);
    1945           0 :                 return NULL;
    1946             :         }
    1947             : 
    1948          12 :         fs_channel_create(fs, &ctx->ch, 512);
    1949             : 
    1950          12 :         ctx->ch.send_request = fs->send_request;
    1951          12 :         ctx->ch.sync = 1;
    1952             : 
    1953          12 :         return ctx;
    1954             : }
    1955             : 
    1956             : 
    1957             : void
    1958          12 : spdk_fs_free_thread_ctx(struct spdk_fs_thread_ctx *ctx)
    1959             : {
    1960          12 :         assert(ctx->ch.sync == 1);
    1961             : 
    1962             :         while (true) {
    1963          12 :                 pthread_spin_lock(&ctx->ch.lock);
    1964          12 :                 if (ctx->ch.outstanding_reqs == 0) {
    1965          12 :                         pthread_spin_unlock(&ctx->ch.lock);
    1966          12 :                         break;
    1967             :                 }
    1968           0 :                 pthread_spin_unlock(&ctx->ch.lock);
    1969           0 :                 usleep(1000);
    1970             :         }
    1971             : 
    1972          12 :         fs_channel_destroy(NULL, &ctx->ch);
    1973          12 :         free(ctx);
    1974          12 : }
    1975             : 
    1976             : int
    1977           0 : spdk_fs_set_cache_size(uint64_t size_in_mb)
    1978             : {
    1979             :         /* setting g_fs_cache_size is only permitted if cache pool
    1980             :          * is already freed or hasn't been initialized
    1981             :          */
    1982           0 :         if (g_cache_pool != NULL) {
    1983           0 :                 return -EPERM;
    1984             :         }
    1985             : 
    1986           0 :         g_fs_cache_size = size_in_mb * 1024 * 1024;
    1987             : 
    1988           0 :         return 0;
    1989             : }
    1990             : 
    1991             : uint64_t
    1992           0 : spdk_fs_get_cache_size(void)
    1993             : {
    1994           0 :         return g_fs_cache_size / (1024 * 1024);
    1995             : }
    1996             : 
    1997             : static void __file_flush(void *ctx);
    1998             : 
    1999             : /* Try to free some cache buffers from this file.
    2000             :  */
    2001             : static int
    2002           0 : reclaim_cache_buffers(struct spdk_file *file)
    2003             : {
    2004             :         int rc;
    2005             : 
    2006           0 :         BLOBFS_TRACE(file, "free=%s\n", file->name);
    2007             : 
    2008             :         /* The function is safe to be called with any threads, while the file
    2009             :          * lock maybe locked by other thread for now, so try to get the file
    2010             :          * lock here.
    2011             :          */
    2012           0 :         rc = pthread_spin_trylock(&file->lock);
    2013           0 :         if (rc != 0) {
    2014           0 :                 return -1;
    2015             :         }
    2016             : 
    2017           0 :         if (file->tree->present_mask == 0) {
    2018           0 :                 pthread_spin_unlock(&file->lock);
    2019           0 :                 return -1;
    2020             :         }
    2021           0 :         tree_free_buffers(file->tree);
    2022             : 
    2023           0 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2024             :         /* If not freed, put it in the end of the queue */
    2025           0 :         if (file->tree->present_mask != 0) {
    2026           0 :                 TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
    2027             :         }
    2028             : 
    2029             :         /* tree_free_buffers() may have freed the buffer pointed to by file->last.
    2030             :          * So check if current append_pos is still in the cache, and if not, clear
    2031             :          * file->last.
    2032             :          */
    2033           0 :         if (tree_find_buffer(file->tree, file->append_pos) == NULL) {
    2034           0 :                 file->last = NULL;
    2035             :         }
    2036             : 
    2037           0 :         pthread_spin_unlock(&file->lock);
    2038             : 
    2039           0 :         return 0;
    2040             : }
    2041             : 
    2042             : static int
    2043           0 : _blobfs_cache_pool_reclaim(void *arg)
    2044             : {
    2045             :         struct spdk_file *file, *tmp;
    2046             :         int rc;
    2047             : 
    2048           0 :         if (!blobfs_cache_pool_need_reclaim()) {
    2049           0 :                 return SPDK_POLLER_IDLE;
    2050             :         }
    2051             : 
    2052           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2053           0 :                 if (!file->open_for_writing &&
    2054           0 :                     file->priority == SPDK_FILE_PRIORITY_LOW) {
    2055           0 :                         rc = reclaim_cache_buffers(file);
    2056           0 :                         if (rc < 0) {
    2057           0 :                                 continue;
    2058             :                         }
    2059           0 :                         if (!blobfs_cache_pool_need_reclaim()) {
    2060           0 :                                 return SPDK_POLLER_BUSY;
    2061             :                         }
    2062           0 :                         break;
    2063             :                 }
    2064             :         }
    2065             : 
    2066           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2067           0 :                 if (!file->open_for_writing) {
    2068           0 :                         rc = reclaim_cache_buffers(file);
    2069           0 :                         if (rc < 0) {
    2070           0 :                                 continue;
    2071             :                         }
    2072           0 :                         if (!blobfs_cache_pool_need_reclaim()) {
    2073           0 :                                 return SPDK_POLLER_BUSY;
    2074             :                         }
    2075           0 :                         break;
    2076             :                 }
    2077             :         }
    2078             : 
    2079           0 :         TAILQ_FOREACH_SAFE(file, &g_caches, cache_tailq, tmp) {
    2080           0 :                 rc = reclaim_cache_buffers(file);
    2081           0 :                 if (rc < 0) {
    2082           0 :                         continue;
    2083             :                 }
    2084           0 :                 break;
    2085             :         }
    2086             : 
    2087           0 :         return SPDK_POLLER_BUSY;
    2088             : }
    2089             : 
    2090             : static void
    2091           5 : _add_file_to_cache_pool(void *ctx)
    2092             : {
    2093           5 :         struct spdk_file *file = ctx;
    2094             : 
    2095           5 :         TAILQ_INSERT_TAIL(&g_caches, file, cache_tailq);
    2096           5 : }
    2097             : 
    2098             : static void
    2099           0 : _remove_file_from_cache_pool(void *ctx)
    2100             : {
    2101           0 :         struct spdk_file *file = ctx;
    2102             : 
    2103           0 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2104           0 : }
    2105             : 
    2106             : static struct cache_buffer *
    2107          10 : cache_insert_buffer(struct spdk_file *file, uint64_t offset)
    2108             : {
    2109             :         struct cache_buffer *buf;
    2110          10 :         int count = 0;
    2111          10 :         bool need_update = false;
    2112             : 
    2113          10 :         buf = calloc(1, sizeof(*buf));
    2114          10 :         if (buf == NULL) {
    2115           0 :                 SPDK_DEBUGLOG(blobfs, "calloc failed\n");
    2116           0 :                 return NULL;
    2117             :         }
    2118             : 
    2119             :         do {
    2120          10 :                 buf->buf = spdk_mempool_get(g_cache_pool);
    2121          10 :                 if (buf->buf) {
    2122          10 :                         break;
    2123             :                 }
    2124           0 :                 if (count++ == 100) {
    2125           0 :                         SPDK_ERRLOG("Could not allocate cache buffer for file=%p on offset=%jx\n",
    2126             :                                     file, offset);
    2127           0 :                         free(buf);
    2128           0 :                         return NULL;
    2129             :                 }
    2130           0 :                 usleep(BLOBFS_CACHE_POOL_POLL_PERIOD_IN_US);
    2131             :         } while (true);
    2132             : 
    2133          10 :         buf->buf_size = CACHE_BUFFER_SIZE;
    2134          10 :         buf->offset = offset;
    2135             : 
    2136          10 :         if (file->tree->present_mask == 0) {
    2137           5 :                 need_update = true;
    2138             :         }
    2139          10 :         file->tree = tree_insert_buffer(file->tree, buf);
    2140             : 
    2141          10 :         if (need_update) {
    2142           5 :                 spdk_thread_send_msg(g_cache_pool_thread, _add_file_to_cache_pool, file);
    2143             :         }
    2144             : 
    2145          10 :         return buf;
    2146             : }
    2147             : 
    2148             : static struct cache_buffer *
    2149          10 : cache_append_buffer(struct spdk_file *file)
    2150             : {
    2151             :         struct cache_buffer *last;
    2152             : 
    2153          10 :         assert(file->last == NULL || file->last->bytes_filled == file->last->buf_size);
    2154          10 :         assert((file->append_pos % CACHE_BUFFER_SIZE) == 0);
    2155             : 
    2156          10 :         last = cache_insert_buffer(file, file->append_pos);
    2157          10 :         if (last == NULL) {
    2158           0 :                 SPDK_DEBUGLOG(blobfs, "cache_insert_buffer failed\n");
    2159           0 :                 return NULL;
    2160             :         }
    2161             : 
    2162          10 :         file->last = last;
    2163             : 
    2164          10 :         return last;
    2165             : }
    2166             : 
    2167             : static void __check_sync_reqs(struct spdk_file *file);
    2168             : 
    2169             : static void
    2170           7 : __file_cache_finish_sync(void *ctx, int bserrno)
    2171             : {
    2172             :         struct spdk_file *file;
    2173           7 :         struct spdk_fs_request *sync_req = ctx;
    2174             :         struct spdk_fs_cb_args *sync_args;
    2175             : 
    2176           7 :         sync_args = &sync_req->args;
    2177           7 :         file = sync_args->file;
    2178           7 :         pthread_spin_lock(&file->lock);
    2179           7 :         file->length_xattr = sync_args->op.sync.length;
    2180           7 :         assert(sync_args->op.sync.offset <= file->length_flushed);
    2181           7 :         spdk_trace_record(TRACE_BLOBFS_XATTR_END, 0, sync_args->op.sync.offset,
    2182             :                           0, file->name);
    2183           7 :         BLOBFS_TRACE(file, "sync done offset=%jx\n", sync_args->op.sync.offset);
    2184           7 :         TAILQ_REMOVE(&file->sync_requests, sync_req, args.op.sync.tailq);
    2185           7 :         pthread_spin_unlock(&file->lock);
    2186             : 
    2187           7 :         sync_args->fn.file_op(sync_args->arg, bserrno);
    2188             : 
    2189           7 :         free_fs_request(sync_req);
    2190           7 :         __check_sync_reqs(file);
    2191           7 : }
    2192             : 
    2193             : static void
    2194          24 : __check_sync_reqs(struct spdk_file *file)
    2195             : {
    2196             :         struct spdk_fs_request *sync_req;
    2197             : 
    2198          24 :         pthread_spin_lock(&file->lock);
    2199             : 
    2200          24 :         TAILQ_FOREACH(sync_req, &file->sync_requests, args.op.sync.tailq) {
    2201          12 :                 if (sync_req->args.op.sync.offset <= file->length_flushed) {
    2202          12 :                         break;
    2203             :                 }
    2204             :         }
    2205             : 
    2206          24 :         if (sync_req != NULL && !sync_req->args.op.sync.xattr_in_progress) {
    2207           7 :                 BLOBFS_TRACE(file, "set xattr length 0x%jx\n", file->length_flushed);
    2208           7 :                 sync_req->args.op.sync.xattr_in_progress = true;
    2209           7 :                 sync_req->args.op.sync.length = file->length_flushed;
    2210           7 :                 spdk_blob_set_xattr(file->blob, "length", &file->length_flushed,
    2211             :                                     sizeof(file->length_flushed));
    2212             : 
    2213           7 :                 pthread_spin_unlock(&file->lock);
    2214           7 :                 spdk_trace_record(TRACE_BLOBFS_XATTR_START, 0, file->length_flushed,
    2215             :                                   0, file->name);
    2216           7 :                 spdk_blob_sync_md(file->blob, __file_cache_finish_sync, sync_req);
    2217             :         } else {
    2218          17 :                 pthread_spin_unlock(&file->lock);
    2219             :         }
    2220          24 : }
    2221             : 
    2222             : static void
    2223          10 : __file_flush_done(void *ctx, int bserrno)
    2224             : {
    2225          10 :         struct spdk_fs_request *req = ctx;
    2226          10 :         struct spdk_fs_cb_args *args = &req->args;
    2227          10 :         struct spdk_file *file = args->file;
    2228          10 :         struct cache_buffer *next = args->op.flush.cache_buffer;
    2229             : 
    2230          10 :         BLOBFS_TRACE(file, "length=%jx\n", args->op.flush.length);
    2231             : 
    2232          10 :         pthread_spin_lock(&file->lock);
    2233          10 :         next->in_progress = false;
    2234          10 :         next->bytes_flushed += args->op.flush.length;
    2235          10 :         file->length_flushed += args->op.flush.length;
    2236          10 :         if (file->length_flushed > file->length) {
    2237           0 :                 file->length = file->length_flushed;
    2238             :         }
    2239          10 :         if (next->bytes_flushed == next->buf_size) {
    2240           5 :                 BLOBFS_TRACE(file, "write buffer fully flushed 0x%jx\n", file->length_flushed);
    2241           5 :                 next = tree_find_buffer(file->tree, file->length_flushed);
    2242             :         }
    2243             : 
    2244             :         /*
    2245             :          * Assert that there is no cached data that extends past the end of the underlying
    2246             :          *  blob.
    2247             :          */
    2248          10 :         assert(next == NULL || next->offset < __file_get_blob_size(file) ||
    2249             :                next->bytes_filled == 0);
    2250             : 
    2251          10 :         pthread_spin_unlock(&file->lock);
    2252             : 
    2253          10 :         __check_sync_reqs(file);
    2254             : 
    2255          10 :         __file_flush(req);
    2256          10 : }
    2257             : 
    2258             : static void
    2259          20 : __file_flush(void *ctx)
    2260             : {
    2261          20 :         struct spdk_fs_request *req = ctx;
    2262          20 :         struct spdk_fs_cb_args *args = &req->args;
    2263          20 :         struct spdk_file *file = args->file;
    2264             :         struct cache_buffer *next;
    2265          20 :         uint64_t offset, length, start_lba, num_lba;
    2266          20 :         uint32_t lba_size;
    2267             : 
    2268          20 :         pthread_spin_lock(&file->lock);
    2269          20 :         next = tree_find_buffer(file->tree, file->length_flushed);
    2270          20 :         if (next == NULL || next->in_progress ||
    2271          19 :             ((next->bytes_filled < next->buf_size) && TAILQ_EMPTY(&file->sync_requests))) {
    2272             :                 /*
    2273             :                  * There is either no data to flush, a flush I/O is already in
    2274             :                  *  progress, or the next buffer is partially filled but there's no
    2275             :                  *  outstanding request to sync it.
    2276             :                  * So return immediately - if a flush I/O is in progress we will flush
    2277             :                  *  more data after that is completed, or a partial buffer will get flushed
    2278             :                  *  when it is either filled or the file is synced.
    2279             :                  */
    2280           4 :                 free_fs_request(req);
    2281           4 :                 if (next == NULL) {
    2282             :                         /*
    2283             :                          * For cases where a file's cache was evicted, and then the
    2284             :                          *  file was later appended, we will write the data directly
    2285             :                          *  to disk and bypass cache.  So just update length_flushed
    2286             :                          *  here to reflect that all data was already written to disk.
    2287             :                          */
    2288           1 :                         file->length_flushed = file->append_pos;
    2289             :                 }
    2290           4 :                 pthread_spin_unlock(&file->lock);
    2291           4 :                 if (next == NULL) {
    2292             :                         /*
    2293             :                          * There is no data to flush, but we still need to check for any
    2294             :                          *  outstanding sync requests to make sure metadata gets updated.
    2295             :                          */
    2296           1 :                         __check_sync_reqs(file);
    2297             :                 }
    2298           4 :                 return;
    2299             :         }
    2300             : 
    2301          16 :         offset = next->offset + next->bytes_flushed;
    2302          16 :         length = next->bytes_filled - next->bytes_flushed;
    2303          16 :         if (length == 0) {
    2304           6 :                 free_fs_request(req);
    2305           6 :                 pthread_spin_unlock(&file->lock);
    2306             :                 /*
    2307             :                  * There is no data to flush, but we still need to check for any
    2308             :                  *  outstanding sync requests to make sure metadata gets updated.
    2309             :                  */
    2310           6 :                 __check_sync_reqs(file);
    2311           6 :                 return;
    2312             :         }
    2313          10 :         args->op.flush.length = length;
    2314          10 :         args->op.flush.cache_buffer = next;
    2315             : 
    2316          10 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    2317             : 
    2318          10 :         next->in_progress = true;
    2319          10 :         BLOBFS_TRACE(file, "offset=0x%jx length=0x%jx page start=0x%jx num=0x%jx\n",
    2320             :                      offset, length, start_lba, num_lba);
    2321          10 :         pthread_spin_unlock(&file->lock);
    2322          10 :         spdk_blob_io_write(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
    2323          10 :                            next->buf + (start_lba * lba_size) - next->offset,
    2324             :                            start_lba, num_lba, __file_flush_done, req);
    2325             : }
    2326             : 
    2327             : static void
    2328           0 : __file_extend_done(void *arg, int bserrno)
    2329             : {
    2330           0 :         struct spdk_fs_cb_args *args = arg;
    2331             : 
    2332           0 :         __wake_caller(args, bserrno);
    2333           0 : }
    2334             : 
    2335             : static void
    2336           0 : __file_extend_resize_cb(void *_args, int bserrno)
    2337             : {
    2338           0 :         struct spdk_fs_cb_args *args = _args;
    2339           0 :         struct spdk_file *file = args->file;
    2340             : 
    2341           0 :         if (bserrno) {
    2342           0 :                 __wake_caller(args, bserrno);
    2343           0 :                 return;
    2344             :         }
    2345             : 
    2346           0 :         spdk_blob_sync_md(file->blob, __file_extend_done, args);
    2347             : }
    2348             : 
    2349             : static void
    2350           0 : __file_extend_blob(void *_args)
    2351             : {
    2352           0 :         struct spdk_fs_cb_args *args = _args;
    2353           0 :         struct spdk_file *file = args->file;
    2354             : 
    2355           0 :         spdk_blob_resize(file->blob, args->op.resize.num_clusters, __file_extend_resize_cb, args);
    2356           0 : }
    2357             : 
    2358             : static void
    2359           1 : __rw_from_file_done(void *ctx, int bserrno)
    2360             : {
    2361           1 :         struct spdk_fs_request *req = ctx;
    2362             : 
    2363           1 :         __wake_caller(&req->args, bserrno);
    2364           1 :         free_fs_request(req);
    2365           1 : }
    2366             : 
    2367             : static void
    2368           1 : __rw_from_file(void *ctx)
    2369             : {
    2370           1 :         struct spdk_fs_request *req = ctx;
    2371           1 :         struct spdk_fs_cb_args *args = &req->args;
    2372           1 :         struct spdk_file *file = args->file;
    2373             : 
    2374           1 :         if (args->op.rw.is_read) {
    2375           0 :                 spdk_file_read_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
    2376           0 :                                      args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
    2377             :                                      __rw_from_file_done, req);
    2378             :         } else {
    2379           1 :                 spdk_file_write_async(file, file->fs->sync_target.sync_io_channel, args->iovs[0].iov_base,
    2380           1 :                                       args->op.rw.offset, (uint64_t)args->iovs[0].iov_len,
    2381             :                                       __rw_from_file_done, req);
    2382             :         }
    2383           1 : }
    2384             : 
    2385             : struct rw_from_file_arg {
    2386             :         struct spdk_fs_channel *channel;
    2387             :         int rwerrno;
    2388             : };
    2389             : 
    2390             : static int
    2391           1 : __send_rw_from_file(struct spdk_file *file, void *payload,
    2392             :                     uint64_t offset, uint64_t length, bool is_read,
    2393             :                     struct rw_from_file_arg *arg)
    2394             : {
    2395             :         struct spdk_fs_request *req;
    2396             :         struct spdk_fs_cb_args *args;
    2397             : 
    2398           1 :         req = alloc_fs_request_with_iov(arg->channel, 1);
    2399           1 :         if (req == NULL) {
    2400           0 :                 sem_post(&arg->channel->sem);
    2401           0 :                 return -ENOMEM;
    2402             :         }
    2403             : 
    2404           1 :         args = &req->args;
    2405           1 :         args->file = file;
    2406           1 :         args->sem = &arg->channel->sem;
    2407           1 :         args->iovs[0].iov_base = payload;
    2408           1 :         args->iovs[0].iov_len = (size_t)length;
    2409           1 :         args->op.rw.offset = offset;
    2410           1 :         args->op.rw.is_read = is_read;
    2411           1 :         args->rwerrno = &arg->rwerrno;
    2412           1 :         file->fs->send_request(__rw_from_file, req);
    2413           1 :         return 0;
    2414             : }
    2415             : 
    2416             : int
    2417          11 : spdk_file_write(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    2418             :                 void *payload, uint64_t offset, uint64_t length)
    2419             : {
    2420          11 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2421             :         struct spdk_fs_request *flush_req;
    2422             :         uint64_t rem_length, copy, blob_size, cluster_sz;
    2423          11 :         uint32_t cache_buffers_filled = 0;
    2424             :         uint8_t *cur_payload;
    2425             :         struct cache_buffer *last;
    2426             : 
    2427          11 :         BLOBFS_TRACE_RW(file, "offset=%jx length=%jx\n", offset, length);
    2428             : 
    2429          11 :         if (length == 0) {
    2430           1 :                 return 0;
    2431             :         }
    2432             : 
    2433          10 :         if (offset != file->append_pos) {
    2434           0 :                 BLOBFS_TRACE(file, " error offset=%jx append_pos=%jx\n", offset, file->append_pos);
    2435           0 :                 return -EINVAL;
    2436             :         }
    2437             : 
    2438          10 :         pthread_spin_lock(&file->lock);
    2439          10 :         file->open_for_writing = true;
    2440             : 
    2441             :         do {
    2442          10 :                 if ((file->last == NULL) && (file->append_pos % CACHE_BUFFER_SIZE == 0)) {
    2443           5 :                         cache_append_buffer(file);
    2444             :                 }
    2445             : 
    2446          10 :                 if (file->last == NULL) {
    2447           1 :                         struct rw_from_file_arg arg = {};
    2448             :                         int rc;
    2449             : 
    2450           1 :                         arg.channel = channel;
    2451           1 :                         arg.rwerrno = 0;
    2452           1 :                         file->append_pos += length;
    2453           1 :                         pthread_spin_unlock(&file->lock);
    2454           1 :                         rc = __send_rw_from_file(file, payload, offset, length, false, &arg);
    2455           1 :                         if (rc != 0) {
    2456           0 :                                 return rc;
    2457             :                         }
    2458           1 :                         sem_wait(&channel->sem);
    2459           1 :                         return arg.rwerrno;
    2460             :                 }
    2461             : 
    2462           9 :                 blob_size = __file_get_blob_size(file);
    2463             : 
    2464           9 :                 if ((offset + length) > blob_size) {
    2465           0 :                         struct spdk_fs_cb_args extend_args = {};
    2466             : 
    2467           0 :                         cluster_sz = file->fs->bs_opts.cluster_sz;
    2468           0 :                         extend_args.sem = &channel->sem;
    2469           0 :                         extend_args.op.resize.num_clusters = __bytes_to_clusters((offset + length), cluster_sz);
    2470           0 :                         extend_args.file = file;
    2471           0 :                         BLOBFS_TRACE(file, "start resize to %u clusters\n", extend_args.op.resize.num_clusters);
    2472           0 :                         pthread_spin_unlock(&file->lock);
    2473           0 :                         file->fs->send_request(__file_extend_blob, &extend_args);
    2474           0 :                         sem_wait(&channel->sem);
    2475           0 :                         if (extend_args.rc) {
    2476           0 :                                 return extend_args.rc;
    2477             :                         }
    2478           0 :                         pthread_spin_lock(&file->lock);
    2479             :                 }
    2480           9 :         } while (file->last == NULL);
    2481             : 
    2482           9 :         flush_req = alloc_fs_request(channel);
    2483           9 :         if (flush_req == NULL) {
    2484           0 :                 pthread_spin_unlock(&file->lock);
    2485           0 :                 return -ENOMEM;
    2486             :         }
    2487             : 
    2488           9 :         last = file->last;
    2489           9 :         rem_length = length;
    2490           9 :         cur_payload = payload;
    2491          22 :         while (rem_length > 0) {
    2492          13 :                 copy = last->buf_size - last->bytes_filled;
    2493          13 :                 if (copy > rem_length) {
    2494           8 :                         copy = rem_length;
    2495             :                 }
    2496          13 :                 BLOBFS_TRACE_RW(file, "  fill offset=%jx length=%jx\n", file->append_pos, copy);
    2497          13 :                 memcpy(&last->buf[last->bytes_filled], cur_payload, copy);
    2498          13 :                 file->append_pos += copy;
    2499          13 :                 if (file->length < file->append_pos) {
    2500          12 :                         file->length = file->append_pos;
    2501             :                 }
    2502          13 :                 cur_payload += copy;
    2503          13 :                 last->bytes_filled += copy;
    2504          13 :                 rem_length -= copy;
    2505          13 :                 if (last->bytes_filled == last->buf_size) {
    2506           5 :                         cache_buffers_filled++;
    2507           5 :                         last = cache_append_buffer(file);
    2508           5 :                         if (last == NULL) {
    2509           0 :                                 BLOBFS_TRACE(file, "nomem\n");
    2510           0 :                                 free_fs_request(flush_req);
    2511           0 :                                 pthread_spin_unlock(&file->lock);
    2512           0 :                                 return -ENOMEM;
    2513             :                         }
    2514             :                 }
    2515             :         }
    2516             : 
    2517           9 :         pthread_spin_unlock(&file->lock);
    2518             : 
    2519           9 :         if (cache_buffers_filled == 0) {
    2520           6 :                 free_fs_request(flush_req);
    2521           6 :                 return 0;
    2522             :         }
    2523             : 
    2524           3 :         flush_req->args.file = file;
    2525           3 :         file->fs->send_request(__file_flush, flush_req);
    2526           3 :         return 0;
    2527             : }
    2528             : 
    2529             : static void
    2530           0 : __readahead_done(void *ctx, int bserrno)
    2531             : {
    2532           0 :         struct spdk_fs_request *req = ctx;
    2533           0 :         struct spdk_fs_cb_args *args = &req->args;
    2534           0 :         struct cache_buffer *cache_buffer = args->op.readahead.cache_buffer;
    2535           0 :         struct spdk_file *file = args->file;
    2536             : 
    2537           0 :         BLOBFS_TRACE(file, "offset=%jx\n", cache_buffer->offset);
    2538             : 
    2539           0 :         pthread_spin_lock(&file->lock);
    2540           0 :         cache_buffer->bytes_filled = args->op.readahead.length;
    2541           0 :         cache_buffer->bytes_flushed = args->op.readahead.length;
    2542           0 :         cache_buffer->in_progress = false;
    2543           0 :         pthread_spin_unlock(&file->lock);
    2544             : 
    2545           0 :         free_fs_request(req);
    2546           0 : }
    2547             : 
    2548             : static void
    2549           0 : __readahead(void *ctx)
    2550             : {
    2551           0 :         struct spdk_fs_request *req = ctx;
    2552           0 :         struct spdk_fs_cb_args *args = &req->args;
    2553           0 :         struct spdk_file *file = args->file;
    2554           0 :         uint64_t offset, length, start_lba, num_lba;
    2555           0 :         uint32_t lba_size;
    2556             : 
    2557           0 :         offset = args->op.readahead.offset;
    2558           0 :         length = args->op.readahead.length;
    2559           0 :         assert(length > 0);
    2560             : 
    2561           0 :         __get_page_parameters(file, offset, length, &start_lba, &lba_size, &num_lba);
    2562             : 
    2563           0 :         BLOBFS_TRACE(file, "offset=%jx length=%jx page start=%jx num=%jx\n",
    2564             :                      offset, length, start_lba, num_lba);
    2565           0 :         spdk_blob_io_read(file->blob, file->fs->sync_target.sync_fs_channel->bs_channel,
    2566           0 :                           args->op.readahead.cache_buffer->buf,
    2567             :                           start_lba, num_lba, __readahead_done, req);
    2568           0 : }
    2569             : 
    2570             : static uint64_t
    2571           0 : __next_cache_buffer_offset(uint64_t offset)
    2572             : {
    2573           0 :         return (offset + CACHE_BUFFER_SIZE) & ~(CACHE_TREE_LEVEL_MASK(0));
    2574             : }
    2575             : 
    2576             : static void
    2577           0 : check_readahead(struct spdk_file *file, uint64_t offset,
    2578             :                 struct spdk_fs_channel *channel)
    2579             : {
    2580             :         struct spdk_fs_request *req;
    2581             :         struct spdk_fs_cb_args *args;
    2582             : 
    2583           0 :         offset = __next_cache_buffer_offset(offset);
    2584           0 :         if (tree_find_buffer(file->tree, offset) != NULL || file->length <= offset) {
    2585           0 :                 return;
    2586             :         }
    2587             : 
    2588           0 :         req = alloc_fs_request(channel);
    2589           0 :         if (req == NULL) {
    2590           0 :                 return;
    2591             :         }
    2592           0 :         args = &req->args;
    2593             : 
    2594           0 :         BLOBFS_TRACE(file, "offset=%jx\n", offset);
    2595             : 
    2596           0 :         args->file = file;
    2597           0 :         args->op.readahead.offset = offset;
    2598           0 :         args->op.readahead.cache_buffer = cache_insert_buffer(file, offset);
    2599           0 :         if (!args->op.readahead.cache_buffer) {
    2600           0 :                 BLOBFS_TRACE(file, "Cannot allocate buf for offset=%jx\n", offset);
    2601           0 :                 free_fs_request(req);
    2602           0 :                 return;
    2603             :         }
    2604             : 
    2605           0 :         args->op.readahead.cache_buffer->in_progress = true;
    2606           0 :         if (file->length < (offset + CACHE_BUFFER_SIZE)) {
    2607           0 :                 args->op.readahead.length = file->length & (CACHE_BUFFER_SIZE - 1);
    2608             :         } else {
    2609           0 :                 args->op.readahead.length = CACHE_BUFFER_SIZE;
    2610             :         }
    2611           0 :         file->fs->send_request(__readahead, req);
    2612             : }
    2613             : 
    2614             : int64_t
    2615           1 : spdk_file_read(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx,
    2616             :                void *payload, uint64_t offset, uint64_t length)
    2617             : {
    2618           1 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2619             :         uint64_t final_offset, final_length;
    2620           1 :         uint32_t sub_reads = 0;
    2621             :         struct cache_buffer *buf;
    2622             :         uint64_t read_len;
    2623           1 :         struct rw_from_file_arg arg = {};
    2624             : 
    2625           1 :         pthread_spin_lock(&file->lock);
    2626             : 
    2627           1 :         BLOBFS_TRACE_RW(file, "offset=%ju length=%ju\n", offset, length);
    2628             : 
    2629           1 :         file->open_for_writing = false;
    2630             : 
    2631           1 :         if (length == 0 || offset >= file->append_pos) {
    2632           0 :                 pthread_spin_unlock(&file->lock);
    2633           0 :                 return 0;
    2634             :         }
    2635             : 
    2636           1 :         if (offset + length > file->append_pos) {
    2637           0 :                 length = file->append_pos - offset;
    2638             :         }
    2639             : 
    2640           1 :         if (offset != file->next_seq_offset) {
    2641           0 :                 file->seq_byte_count = 0;
    2642             :         }
    2643           1 :         file->seq_byte_count += length;
    2644           1 :         file->next_seq_offset = offset + length;
    2645           1 :         if (file->seq_byte_count >= CACHE_READAHEAD_THRESHOLD) {
    2646           0 :                 check_readahead(file, offset, channel);
    2647           0 :                 check_readahead(file, offset + CACHE_BUFFER_SIZE, channel);
    2648             :         }
    2649             : 
    2650           1 :         arg.channel = channel;
    2651           1 :         arg.rwerrno = 0;
    2652           1 :         final_length = 0;
    2653           1 :         final_offset = offset + length;
    2654           2 :         while (offset < final_offset) {
    2655           1 :                 int ret = 0;
    2656           1 :                 length = NEXT_CACHE_BUFFER_OFFSET(offset) - offset;
    2657           1 :                 if (length > (final_offset - offset)) {
    2658           1 :                         length = final_offset - offset;
    2659             :                 }
    2660             : 
    2661           1 :                 buf = tree_find_filled_buffer(file->tree, offset);
    2662           1 :                 if (buf == NULL) {
    2663           0 :                         pthread_spin_unlock(&file->lock);
    2664           0 :                         ret = __send_rw_from_file(file, payload, offset, length, true, &arg);
    2665           0 :                         pthread_spin_lock(&file->lock);
    2666           0 :                         if (ret == 0) {
    2667           0 :                                 sub_reads++;
    2668             :                         }
    2669             :                 } else {
    2670           1 :                         read_len = length;
    2671           1 :                         if ((offset + length) > (buf->offset + buf->bytes_filled)) {
    2672           0 :                                 read_len = buf->offset + buf->bytes_filled - offset;
    2673             :                         }
    2674           1 :                         BLOBFS_TRACE(file, "read %p offset=%ju length=%ju\n", payload, offset, read_len);
    2675           1 :                         memcpy(payload, &buf->buf[offset - buf->offset], read_len);
    2676           1 :                         if ((offset + read_len) % CACHE_BUFFER_SIZE == 0) {
    2677           0 :                                 tree_remove_buffer(file->tree, buf);
    2678           0 :                                 if (file->tree->present_mask == 0) {
    2679           0 :                                         spdk_thread_send_msg(g_cache_pool_thread, _remove_file_from_cache_pool, file);
    2680             :                                 }
    2681             :                         }
    2682             :                 }
    2683             : 
    2684           1 :                 if (ret == 0) {
    2685           1 :                         final_length += length;
    2686             :                 } else {
    2687           0 :                         arg.rwerrno = ret;
    2688           0 :                         break;
    2689             :                 }
    2690           1 :                 payload += length;
    2691           1 :                 offset += length;
    2692             :         }
    2693           1 :         pthread_spin_unlock(&file->lock);
    2694           1 :         while (sub_reads > 0) {
    2695           0 :                 sem_wait(&channel->sem);
    2696           0 :                 sub_reads--;
    2697             :         }
    2698           1 :         if (arg.rwerrno == 0) {
    2699           1 :                 return final_length;
    2700             :         } else {
    2701           0 :                 return arg.rwerrno;
    2702             :         }
    2703             : }
    2704             : 
    2705             : static void
    2706          18 : _file_sync(struct spdk_file *file, struct spdk_fs_channel *channel,
    2707             :            spdk_file_op_complete cb_fn, void *cb_arg)
    2708             : {
    2709             :         struct spdk_fs_request *sync_req;
    2710             :         struct spdk_fs_request *flush_req;
    2711             :         struct spdk_fs_cb_args *sync_args;
    2712             :         struct spdk_fs_cb_args *flush_args;
    2713             : 
    2714          18 :         BLOBFS_TRACE(file, "offset=%jx\n", file->append_pos);
    2715             : 
    2716          18 :         pthread_spin_lock(&file->lock);
    2717          18 :         if (file->append_pos <= file->length_xattr) {
    2718          11 :                 BLOBFS_TRACE(file, "done - file already synced\n");
    2719          11 :                 pthread_spin_unlock(&file->lock);
    2720          11 :                 cb_fn(cb_arg, 0);
    2721          11 :                 return;
    2722             :         }
    2723             : 
    2724           7 :         sync_req = alloc_fs_request(channel);
    2725           7 :         if (!sync_req) {
    2726           0 :                 SPDK_ERRLOG("Cannot allocate sync req for file=%s\n", file->name);
    2727           0 :                 pthread_spin_unlock(&file->lock);
    2728           0 :                 cb_fn(cb_arg, -ENOMEM);
    2729           0 :                 return;
    2730             :         }
    2731           7 :         sync_args = &sync_req->args;
    2732             : 
    2733           7 :         flush_req = alloc_fs_request(channel);
    2734           7 :         if (!flush_req) {
    2735           0 :                 SPDK_ERRLOG("Cannot allocate flush req for file=%s\n", file->name);
    2736           0 :                 free_fs_request(sync_req);
    2737           0 :                 pthread_spin_unlock(&file->lock);
    2738           0 :                 cb_fn(cb_arg, -ENOMEM);
    2739           0 :                 return;
    2740             :         }
    2741           7 :         flush_args = &flush_req->args;
    2742             : 
    2743           7 :         sync_args->file = file;
    2744           7 :         sync_args->fn.file_op = cb_fn;
    2745           7 :         sync_args->arg = cb_arg;
    2746           7 :         sync_args->op.sync.offset = file->append_pos;
    2747           7 :         sync_args->op.sync.xattr_in_progress = false;
    2748           7 :         TAILQ_INSERT_TAIL(&file->sync_requests, sync_req, args.op.sync.tailq);
    2749           7 :         pthread_spin_unlock(&file->lock);
    2750             : 
    2751           7 :         flush_args->file = file;
    2752           7 :         channel->send_request(__file_flush, flush_req);
    2753             : }
    2754             : 
    2755             : int
    2756          12 : spdk_file_sync(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
    2757             : {
    2758          12 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2759          12 :         struct spdk_fs_cb_args args = {};
    2760             : 
    2761          12 :         args.sem = &channel->sem;
    2762          12 :         _file_sync(file, channel, __wake_caller, &args);
    2763          12 :         sem_wait(&channel->sem);
    2764             : 
    2765          12 :         return args.rc;
    2766             : }
    2767             : 
    2768             : void
    2769           6 : spdk_file_sync_async(struct spdk_file *file, struct spdk_io_channel *_channel,
    2770             :                      spdk_file_op_complete cb_fn, void *cb_arg)
    2771             : {
    2772           6 :         struct spdk_fs_channel *channel = spdk_io_channel_get_ctx(_channel);
    2773             : 
    2774           6 :         _file_sync(file, channel, cb_fn, cb_arg);
    2775           6 : }
    2776             : 
    2777             : void
    2778           0 : spdk_file_set_priority(struct spdk_file *file, uint32_t priority)
    2779             : {
    2780           0 :         BLOBFS_TRACE(file, "priority=%u\n", priority);
    2781           0 :         file->priority = priority;
    2782             : 
    2783           0 : }
    2784             : 
    2785             : /*
    2786             :  * Close routines
    2787             :  */
    2788             : 
    2789             : static void
    2790          17 : __file_close_async_done(void *ctx, int bserrno)
    2791             : {
    2792          17 :         struct spdk_fs_request *req = ctx;
    2793          17 :         struct spdk_fs_cb_args *args = &req->args;
    2794          17 :         struct spdk_file *file = args->file;
    2795             : 
    2796          17 :         spdk_trace_record(TRACE_BLOBFS_CLOSE, 0, 0, 0, file->name);
    2797             : 
    2798          17 :         if (file->is_deleted) {
    2799           2 :                 spdk_fs_delete_file_async(file->fs, file->name, blob_delete_cb, ctx);
    2800           2 :                 return;
    2801             :         }
    2802             : 
    2803          15 :         args->fn.file_op(args->arg, bserrno);
    2804          15 :         free_fs_request(req);
    2805             : }
    2806             : 
    2807             : static void
    2808          17 : __file_close_async(struct spdk_file *file, struct spdk_fs_request *req)
    2809             : {
    2810             :         struct spdk_blob *blob;
    2811             : 
    2812          17 :         pthread_spin_lock(&file->lock);
    2813          17 :         if (file->ref_count == 0) {
    2814           0 :                 pthread_spin_unlock(&file->lock);
    2815           0 :                 __file_close_async_done(req, -EBADF);
    2816           0 :                 return;
    2817             :         }
    2818             : 
    2819          17 :         file->ref_count--;
    2820          17 :         if (file->ref_count > 0) {
    2821           0 :                 pthread_spin_unlock(&file->lock);
    2822           0 :                 req->args.fn.file_op(req->args.arg, 0);
    2823           0 :                 free_fs_request(req);
    2824           0 :                 return;
    2825             :         }
    2826             : 
    2827          17 :         pthread_spin_unlock(&file->lock);
    2828             : 
    2829          17 :         blob = file->blob;
    2830          17 :         file->blob = NULL;
    2831          17 :         spdk_blob_close(blob, __file_close_async_done, req);
    2832             : }
    2833             : 
    2834             : static void
    2835           6 : __file_close_async__sync_done(void *arg, int fserrno)
    2836             : {
    2837           6 :         struct spdk_fs_request *req = arg;
    2838           6 :         struct spdk_fs_cb_args *args = &req->args;
    2839             : 
    2840           6 :         __file_close_async(args->file, req);
    2841           6 : }
    2842             : 
    2843             : void
    2844           6 : spdk_file_close_async(struct spdk_file *file, spdk_file_op_complete cb_fn, void *cb_arg)
    2845             : {
    2846             :         struct spdk_fs_request *req;
    2847             :         struct spdk_fs_cb_args *args;
    2848             : 
    2849           6 :         req = alloc_fs_request(file->fs->md_target.md_fs_channel);
    2850           6 :         if (req == NULL) {
    2851           0 :                 SPDK_ERRLOG("Cannot allocate close async req for file=%s\n", file->name);
    2852           0 :                 cb_fn(cb_arg, -ENOMEM);
    2853           0 :                 return;
    2854             :         }
    2855             : 
    2856           6 :         args = &req->args;
    2857           6 :         args->file = file;
    2858           6 :         args->fn.file_op = cb_fn;
    2859           6 :         args->arg = cb_arg;
    2860             : 
    2861           6 :         spdk_file_sync_async(file, file->fs->md_target.md_io_channel, __file_close_async__sync_done, req);
    2862             : }
    2863             : 
    2864             : static void
    2865          11 : __file_close(void *arg)
    2866             : {
    2867          11 :         struct spdk_fs_request *req = arg;
    2868          11 :         struct spdk_fs_cb_args *args = &req->args;
    2869          11 :         struct spdk_file *file = args->file;
    2870             : 
    2871          11 :         __file_close_async(file, req);
    2872          11 : }
    2873             : 
    2874             : int
    2875          11 : spdk_file_close(struct spdk_file *file, struct spdk_fs_thread_ctx *ctx)
    2876             : {
    2877          11 :         struct spdk_fs_channel *channel = (struct spdk_fs_channel *)ctx;
    2878             :         struct spdk_fs_request *req;
    2879             :         struct spdk_fs_cb_args *args;
    2880             : 
    2881          11 :         req = alloc_fs_request(channel);
    2882          11 :         if (req == NULL) {
    2883           0 :                 SPDK_ERRLOG("Cannot allocate close req for file=%s\n", file->name);
    2884           0 :                 return -ENOMEM;
    2885             :         }
    2886             : 
    2887          11 :         args = &req->args;
    2888             : 
    2889          11 :         spdk_file_sync(file, ctx);
    2890          11 :         BLOBFS_TRACE(file, "name=%s\n", file->name);
    2891          11 :         args->file = file;
    2892          11 :         args->sem = &channel->sem;
    2893          11 :         args->fn.file_op = __wake_caller;
    2894          11 :         args->arg = args;
    2895          11 :         channel->send_request(__file_close, req);
    2896          11 :         sem_wait(&channel->sem);
    2897             : 
    2898          11 :         return args->rc;
    2899             : }
    2900             : 
    2901             : int
    2902           0 : spdk_file_get_id(struct spdk_file *file, void *id, size_t size)
    2903             : {
    2904           0 :         if (size < sizeof(spdk_blob_id)) {
    2905           0 :                 return -EINVAL;
    2906             :         }
    2907             : 
    2908           0 :         memcpy(id, &file->blobid, sizeof(spdk_blob_id));
    2909             : 
    2910           0 :         return sizeof(spdk_blob_id);
    2911             : }
    2912             : 
    2913             : static void
    2914           5 : _file_free(void *ctx)
    2915             : {
    2916           5 :         struct spdk_file *file = ctx;
    2917             : 
    2918           5 :         TAILQ_REMOVE(&g_caches, file, cache_tailq);
    2919             : 
    2920           5 :         free(file->name);
    2921           5 :         free(file->tree);
    2922           5 :         free(file);
    2923           5 : }
    2924             : 
    2925             : static void
    2926          18 : file_free(struct spdk_file *file)
    2927             : {
    2928          18 :         BLOBFS_TRACE(file, "free=%s\n", file->name);
    2929          18 :         pthread_spin_lock(&file->lock);
    2930          18 :         if (file->tree->present_mask == 0) {
    2931          13 :                 pthread_spin_unlock(&file->lock);
    2932          13 :                 free(file->name);
    2933          13 :                 free(file->tree);
    2934          13 :                 free(file);
    2935          13 :                 return;
    2936             :         }
    2937             : 
    2938           5 :         tree_free_buffers(file->tree);
    2939           5 :         assert(file->tree->present_mask == 0);
    2940           5 :         spdk_thread_send_msg(g_cache_pool_thread, _file_free, file);
    2941           5 :         pthread_spin_unlock(&file->lock);
    2942             : }
    2943             : 
    2944           2 : SPDK_LOG_REGISTER_COMPONENT(blobfs)
    2945           2 : SPDK_LOG_REGISTER_COMPONENT(blobfs_rw)

Generated by: LCOV version 1.15