LCOV - code coverage report
Current view: top level - module/bdev/raid - bdev_raid.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 838 1802 46.5 %
Date: 2024-11-20 17:28:58 Functions: 81 145 55.9 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2018 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  *   Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5             :  */
       6             : 
       7             : #include "bdev_raid.h"
       8             : #include "spdk/env.h"
       9             : #include "spdk/thread.h"
      10             : #include "spdk/log.h"
      11             : #include "spdk/string.h"
      12             : #include "spdk/util.h"
      13             : #include "spdk/json.h"
      14             : #include "spdk/likely.h"
      15             : #include "spdk/trace.h"
      16             : #include "spdk_internal/trace_defs.h"
      17             : 
      18             : #define RAID_OFFSET_BLOCKS_INVALID      UINT64_MAX
      19             : #define RAID_BDEV_PROCESS_MAX_QD        16
      20             : 
      21             : #define RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT        1024
      22             : #define RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT  0
      23             : 
      24             : static bool g_shutdown_started = false;
      25             : 
      26             : /* List of all raid bdevs */
      27             : struct raid_all_tailq g_raid_bdev_list = TAILQ_HEAD_INITIALIZER(g_raid_bdev_list);
      28             : 
      29             : static TAILQ_HEAD(, raid_bdev_module) g_raid_modules = TAILQ_HEAD_INITIALIZER(g_raid_modules);
      30             : 
      31             : /*
      32             :  * raid_bdev_io_channel is the context of spdk_io_channel for raid bdev device. It
      33             :  * contains the relationship of raid bdev io channel with base bdev io channels.
      34             :  */
      35             : struct raid_bdev_io_channel {
      36             :         /* Array of IO channels of base bdevs */
      37             :         struct spdk_io_channel  **base_channel;
      38             : 
      39             :         /* Private raid module IO channel */
      40             :         struct spdk_io_channel  *module_channel;
      41             : 
      42             :         /* Background process data */
      43             :         struct {
      44             :                 uint64_t offset;
      45             :                 struct spdk_io_channel *target_ch;
      46             :                 struct raid_bdev_io_channel *ch_processed;
      47             :         } process;
      48             : };
      49             : 
      50             : enum raid_bdev_process_state {
      51             :         RAID_PROCESS_STATE_INIT,
      52             :         RAID_PROCESS_STATE_RUNNING,
      53             :         RAID_PROCESS_STATE_STOPPING,
      54             :         RAID_PROCESS_STATE_STOPPED,
      55             : };
      56             : 
      57             : struct raid_process_qos {
      58             :         bool enable_qos;
      59             :         uint64_t last_tsc;
      60             :         double bytes_per_tsc;
      61             :         double bytes_available;
      62             :         double bytes_max;
      63             :         struct spdk_poller *process_continue_poller;
      64             : };
      65             : 
      66             : struct raid_bdev_process {
      67             :         struct raid_bdev                *raid_bdev;
      68             :         enum raid_process_type          type;
      69             :         enum raid_bdev_process_state    state;
      70             :         struct spdk_thread              *thread;
      71             :         struct raid_bdev_io_channel     *raid_ch;
      72             :         TAILQ_HEAD(, raid_bdev_process_request) requests;
      73             :         uint64_t                        max_window_size;
      74             :         uint64_t                        window_size;
      75             :         uint64_t                        window_remaining;
      76             :         int                             window_status;
      77             :         uint64_t                        window_offset;
      78             :         bool                            window_range_locked;
      79             :         struct raid_base_bdev_info      *target;
      80             :         int                             status;
      81             :         TAILQ_HEAD(, raid_process_finish_action) finish_actions;
      82             :         struct raid_process_qos         qos;
      83             : };
      84             : 
      85             : struct raid_process_finish_action {
      86             :         spdk_msg_fn cb;
      87             :         void *cb_ctx;
      88             :         TAILQ_ENTRY(raid_process_finish_action) link;
      89             : };
      90             : 
      91             : static struct spdk_raid_bdev_opts g_opts = {
      92             :         .process_window_size_kb = RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT,
      93             :         .process_max_bandwidth_mb_sec = RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT,
      94             : };
      95             : 
      96             : void
      97           0 : raid_bdev_get_opts(struct spdk_raid_bdev_opts *opts)
      98             : {
      99           0 :         *opts = g_opts;
     100           0 : }
     101             : 
     102             : int
     103           1 : raid_bdev_set_opts(const struct spdk_raid_bdev_opts *opts)
     104             : {
     105           1 :         if (opts->process_window_size_kb == 0) {
     106           0 :                 return -EINVAL;
     107             :         }
     108             : 
     109           1 :         g_opts = *opts;
     110             : 
     111           1 :         return 0;
     112             : }
     113             : 
     114             : static struct raid_bdev_module *
     115          19 : raid_bdev_module_find(enum raid_level level)
     116             : {
     117             :         struct raid_bdev_module *raid_module;
     118             : 
     119          20 :         TAILQ_FOREACH(raid_module, &g_raid_modules, link) {
     120          18 :                 if (raid_module->level == level) {
     121          17 :                         return raid_module;
     122             :                 }
     123             :         }
     124             : 
     125           2 :         return NULL;
     126             : }
     127             : 
     128             : void
     129           1 : raid_bdev_module_list_add(struct raid_bdev_module *raid_module)
     130             : {
     131           1 :         if (raid_bdev_module_find(raid_module->level) != NULL) {
     132           0 :                 SPDK_ERRLOG("module for raid level '%s' already registered.\n",
     133             :                             raid_bdev_level_to_str(raid_module->level));
     134           0 :                 assert(false);
     135             :         } else {
     136           1 :                 TAILQ_INSERT_TAIL(&g_raid_modules, raid_module, link);
     137             :         }
     138           1 : }
     139             : 
     140             : struct spdk_io_channel *
     141           0 : raid_bdev_channel_get_base_channel(struct raid_bdev_io_channel *raid_ch, uint8_t idx)
     142             : {
     143           0 :         return raid_ch->base_channel[idx];
     144             : }
     145             : 
     146             : void *
     147           0 : raid_bdev_channel_get_module_ctx(struct raid_bdev_io_channel *raid_ch)
     148             : {
     149           0 :         assert(raid_ch->module_channel != NULL);
     150             : 
     151           0 :         return spdk_io_channel_get_ctx(raid_ch->module_channel);
     152             : }
     153             : 
     154             : struct raid_base_bdev_info *
     155           0 : raid_bdev_channel_get_base_info(struct raid_bdev_io_channel *raid_ch, struct spdk_bdev *base_bdev)
     156             : {
     157           0 :         struct spdk_io_channel *ch = spdk_io_channel_from_ctx(raid_ch);
     158           0 :         struct raid_bdev *raid_bdev = spdk_io_channel_get_io_device(ch);
     159             :         uint8_t i;
     160             : 
     161           0 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     162           0 :                 struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[i];
     163             : 
     164           0 :                 if (base_info->is_configured &&
     165           0 :                     spdk_bdev_desc_get_bdev(base_info->desc) == base_bdev) {
     166           0 :                         return base_info;
     167             :                 }
     168             :         }
     169             : 
     170           0 :         return NULL;
     171             : }
     172             : 
     173             : /* Function declarations */
     174             : static void     raid_bdev_examine(struct spdk_bdev *bdev);
     175             : static int      raid_bdev_init(void);
     176             : static void     raid_bdev_deconfigure(struct raid_bdev *raid_bdev,
     177             :                                       raid_bdev_destruct_cb cb_fn, void *cb_arg);
     178             : 
     179             : static void
     180           7 : raid_bdev_ch_process_cleanup(struct raid_bdev_io_channel *raid_ch)
     181             : {
     182           7 :         raid_ch->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     183             : 
     184           7 :         if (raid_ch->process.target_ch != NULL) {
     185           1 :                 spdk_put_io_channel(raid_ch->process.target_ch);
     186           1 :                 raid_ch->process.target_ch = NULL;
     187             :         }
     188             : 
     189           7 :         if (raid_ch->process.ch_processed != NULL) {
     190           3 :                 free(raid_ch->process.ch_processed->base_channel);
     191           3 :                 free(raid_ch->process.ch_processed);
     192           3 :                 raid_ch->process.ch_processed = NULL;
     193             :         }
     194           7 : }
     195             : 
     196             : static int
     197           3 : raid_bdev_ch_process_setup(struct raid_bdev_io_channel *raid_ch, struct raid_bdev_process *process)
     198             : {
     199           3 :         struct raid_bdev *raid_bdev = process->raid_bdev;
     200             :         struct raid_bdev_io_channel *raid_ch_processed;
     201             :         struct raid_base_bdev_info *base_info;
     202             : 
     203           3 :         raid_ch->process.offset = process->window_offset;
     204             : 
     205             :         /* In the future we may have other types of processes which don't use a target bdev,
     206             :          * like data scrubbing or strip size migration. Until then, expect that there always is
     207             :          * a process target. */
     208           3 :         assert(process->target != NULL);
     209             : 
     210           3 :         raid_ch->process.target_ch = spdk_bdev_get_io_channel(process->target->desc);
     211           3 :         if (raid_ch->process.target_ch == NULL) {
     212           0 :                 goto err;
     213             :         }
     214             : 
     215           3 :         raid_ch_processed = calloc(1, sizeof(*raid_ch_processed));
     216           3 :         if (raid_ch_processed == NULL) {
     217           0 :                 goto err;
     218             :         }
     219           3 :         raid_ch->process.ch_processed = raid_ch_processed;
     220             : 
     221           3 :         raid_ch_processed->base_channel = calloc(raid_bdev->num_base_bdevs,
     222             :                                           sizeof(*raid_ch_processed->base_channel));
     223           3 :         if (raid_ch_processed->base_channel == NULL) {
     224           0 :                 goto err;
     225             :         }
     226             : 
     227          99 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     228          96 :                 uint8_t slot = raid_bdev_base_bdev_slot(base_info);
     229             : 
     230          96 :                 if (base_info != process->target) {
     231          93 :                         raid_ch_processed->base_channel[slot] = raid_ch->base_channel[slot];
     232             :                 } else {
     233           3 :                         raid_ch_processed->base_channel[slot] = raid_ch->process.target_ch;
     234             :                 }
     235             :         }
     236             : 
     237           3 :         raid_ch_processed->module_channel = raid_ch->module_channel;
     238           3 :         raid_ch_processed->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     239             : 
     240           3 :         return 0;
     241           0 : err:
     242           0 :         raid_bdev_ch_process_cleanup(raid_ch);
     243           0 :         return -ENOMEM;
     244             : }
     245             : 
     246             : /*
     247             :  * brief:
     248             :  * raid_bdev_create_cb function is a cb function for raid bdev which creates the
     249             :  * hierarchy from raid bdev to base bdev io channels. It will be called per core
     250             :  * params:
     251             :  * io_device - pointer to raid bdev io device represented by raid_bdev
     252             :  * ctx_buf - pointer to context buffer for raid bdev io channel
     253             :  * returns:
     254             :  * 0 - success
     255             :  * non zero - failure
     256             :  */
     257             : static int
     258           5 : raid_bdev_create_cb(void *io_device, void *ctx_buf)
     259             : {
     260           5 :         struct raid_bdev            *raid_bdev = io_device;
     261           5 :         struct raid_bdev_io_channel *raid_ch = ctx_buf;
     262             :         uint8_t i;
     263           5 :         int ret = -ENOMEM;
     264             : 
     265           5 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_create_cb, %p\n", raid_ch);
     266             : 
     267           5 :         assert(raid_bdev != NULL);
     268           5 :         assert(raid_bdev->state == RAID_BDEV_STATE_ONLINE);
     269             : 
     270           5 :         raid_ch->base_channel = calloc(raid_bdev->num_base_bdevs, sizeof(struct spdk_io_channel *));
     271           5 :         if (!raid_ch->base_channel) {
     272           0 :                 SPDK_ERRLOG("Unable to allocate base bdevs io channel\n");
     273           0 :                 return -ENOMEM;
     274             :         }
     275             : 
     276         165 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     277             :                 /*
     278             :                  * Get the spdk_io_channel for all the base bdevs. This is used during
     279             :                  * split logic to send the respective child bdev ios to respective base
     280             :                  * bdev io channel.
     281             :                  * Skip missing base bdevs and the process target, which should also be treated as
     282             :                  * missing until the process completes.
     283             :                  */
     284         160 :                 if (raid_bdev->base_bdev_info[i].is_configured == false ||
     285         160 :                     raid_bdev->base_bdev_info[i].is_process_target == true) {
     286           0 :                         continue;
     287             :                 }
     288         320 :                 raid_ch->base_channel[i] = spdk_bdev_get_io_channel(
     289         160 :                                                    raid_bdev->base_bdev_info[i].desc);
     290         160 :                 if (!raid_ch->base_channel[i]) {
     291           0 :                         SPDK_ERRLOG("Unable to create io channel for base bdev\n");
     292           0 :                         goto err;
     293             :                 }
     294             :         }
     295             : 
     296           5 :         if (raid_bdev->module->get_io_channel) {
     297           0 :                 raid_ch->module_channel = raid_bdev->module->get_io_channel(raid_bdev);
     298           0 :                 if (!raid_ch->module_channel) {
     299           0 :                         SPDK_ERRLOG("Unable to create io channel for raid module\n");
     300           0 :                         goto err;
     301             :                 }
     302             :         }
     303             : 
     304           5 :         if (raid_bdev->process != NULL) {
     305           3 :                 ret = raid_bdev_ch_process_setup(raid_ch, raid_bdev->process);
     306           3 :                 if (ret != 0) {
     307           0 :                         SPDK_ERRLOG("Failed to setup process io channel\n");
     308           0 :                         goto err;
     309             :                 }
     310             :         } else {
     311           2 :                 raid_ch->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     312             :         }
     313             : 
     314           5 :         return 0;
     315           0 : err:
     316           0 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     317           0 :                 if (raid_ch->base_channel[i] != NULL) {
     318           0 :                         spdk_put_io_channel(raid_ch->base_channel[i]);
     319             :                 }
     320             :         }
     321           0 :         free(raid_ch->base_channel);
     322             : 
     323           0 :         raid_bdev_ch_process_cleanup(raid_ch);
     324             : 
     325           0 :         return ret;
     326             : }
     327             : 
     328             : /*
     329             :  * brief:
     330             :  * raid_bdev_destroy_cb function is a cb function for raid bdev which deletes the
     331             :  * hierarchy from raid bdev to base bdev io channels. It will be called per core
     332             :  * params:
     333             :  * io_device - pointer to raid bdev io device represented by raid_bdev
     334             :  * ctx_buf - pointer to context buffer for raid bdev io channel
     335             :  * returns:
     336             :  * none
     337             :  */
     338             : static void
     339           5 : raid_bdev_destroy_cb(void *io_device, void *ctx_buf)
     340             : {
     341           5 :         struct raid_bdev *raid_bdev = io_device;
     342           5 :         struct raid_bdev_io_channel *raid_ch = ctx_buf;
     343             :         uint8_t i;
     344             : 
     345           5 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_destroy_cb\n");
     346             : 
     347           5 :         assert(raid_ch != NULL);
     348           5 :         assert(raid_ch->base_channel);
     349             : 
     350           5 :         if (raid_ch->module_channel) {
     351           0 :                 spdk_put_io_channel(raid_ch->module_channel);
     352             :         }
     353             : 
     354         165 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     355             :                 /* Free base bdev channels */
     356         160 :                 if (raid_ch->base_channel[i] != NULL) {
     357         160 :                         spdk_put_io_channel(raid_ch->base_channel[i]);
     358             :                 }
     359             :         }
     360           5 :         free(raid_ch->base_channel);
     361           5 :         raid_ch->base_channel = NULL;
     362             : 
     363           5 :         raid_bdev_ch_process_cleanup(raid_ch);
     364           5 : }
     365             : 
     366             : /*
     367             :  * brief:
     368             :  * raid_bdev_cleanup is used to cleanup raid_bdev related data
     369             :  * structures.
     370             :  * params:
     371             :  * raid_bdev - pointer to raid_bdev
     372             :  * returns:
     373             :  * none
     374             :  */
     375             : static void
     376          17 : raid_bdev_cleanup(struct raid_bdev *raid_bdev)
     377             : {
     378             :         struct raid_base_bdev_info *base_info;
     379             : 
     380          17 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_cleanup, %p name %s, state %s\n",
     381             :                       raid_bdev, raid_bdev->bdev.name, raid_bdev_state_to_str(raid_bdev->state));
     382          17 :         assert(raid_bdev->state != RAID_BDEV_STATE_ONLINE);
     383          17 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
     384             : 
     385         561 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     386         544 :                 assert(base_info->desc == NULL);
     387         544 :                 free(base_info->name);
     388             :         }
     389             : 
     390          17 :         TAILQ_REMOVE(&g_raid_bdev_list, raid_bdev, global_link);
     391          17 : }
     392             : 
     393             : static void
     394          17 : raid_bdev_free(struct raid_bdev *raid_bdev)
     395             : {
     396          17 :         raid_bdev_free_superblock(raid_bdev);
     397          17 :         free(raid_bdev->base_bdev_info);
     398          17 :         free(raid_bdev->bdev.name);
     399          17 :         free(raid_bdev);
     400          17 : }
     401             : 
     402             : static void
     403           3 : raid_bdev_cleanup_and_free(struct raid_bdev *raid_bdev)
     404             : {
     405           3 :         raid_bdev_cleanup(raid_bdev);
     406           3 :         raid_bdev_free(raid_bdev);
     407           3 : }
     408             : 
     409             : static void
     410         510 : raid_bdev_deconfigure_base_bdev(struct raid_base_bdev_info *base_info)
     411             : {
     412         510 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
     413             : 
     414         510 :         assert(base_info->is_configured);
     415         510 :         assert(raid_bdev->num_base_bdevs_discovered);
     416         510 :         raid_bdev->num_base_bdevs_discovered--;
     417         510 :         base_info->is_configured = false;
     418         510 :         base_info->is_process_target = false;
     419         510 : }
     420             : 
     421             : /*
     422             :  * brief:
     423             :  * free resource of base bdev for raid bdev
     424             :  * params:
     425             :  * base_info - raid base bdev info
     426             :  * returns:
     427             :  * none
     428             :  */
     429             : static void
     430         543 : raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
     431             : {
     432         543 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
     433             : 
     434         543 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
     435         543 :         assert(base_info->configure_cb == NULL);
     436             : 
     437         543 :         free(base_info->name);
     438         543 :         base_info->name = NULL;
     439         543 :         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
     440         448 :                 spdk_uuid_set_null(&base_info->uuid);
     441             :         }
     442         543 :         base_info->is_failed = false;
     443             : 
     444             :         /* clear `data_offset` to allow it to be recalculated during configuration */
     445         543 :         base_info->data_offset = 0;
     446             : 
     447         543 :         if (base_info->desc == NULL) {
     448          33 :                 return;
     449             :         }
     450             : 
     451         510 :         spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(base_info->desc));
     452         510 :         spdk_bdev_close(base_info->desc);
     453         510 :         base_info->desc = NULL;
     454         510 :         spdk_put_io_channel(base_info->app_thread_ch);
     455         510 :         base_info->app_thread_ch = NULL;
     456             : 
     457         510 :         if (base_info->is_configured) {
     458         510 :                 raid_bdev_deconfigure_base_bdev(base_info);
     459             :         }
     460             : }
     461             : 
     462             : static void
     463          14 : raid_bdev_io_device_unregister_cb(void *io_device)
     464             : {
     465          14 :         struct raid_bdev *raid_bdev = io_device;
     466             : 
     467          14 :         if (raid_bdev->num_base_bdevs_discovered == 0) {
     468             :                 /* Free raid_bdev when there are no base bdevs left */
     469          14 :                 SPDK_DEBUGLOG(bdev_raid, "raid bdev base bdevs is 0, going to free all in destruct\n");
     470          14 :                 raid_bdev_cleanup(raid_bdev);
     471          14 :                 spdk_bdev_destruct_done(&raid_bdev->bdev, 0);
     472          14 :                 raid_bdev_free(raid_bdev);
     473             :         } else {
     474           0 :                 spdk_bdev_destruct_done(&raid_bdev->bdev, 0);
     475             :         }
     476          14 : }
     477             : 
     478             : void
     479          14 : raid_bdev_module_stop_done(struct raid_bdev *raid_bdev)
     480             : {
     481          14 :         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
     482          14 :                 spdk_io_device_unregister(raid_bdev, raid_bdev_io_device_unregister_cb);
     483             :         }
     484          14 : }
     485             : 
     486             : static void
     487          14 : _raid_bdev_destruct(void *ctxt)
     488             : {
     489          14 :         struct raid_bdev *raid_bdev = ctxt;
     490             :         struct raid_base_bdev_info *base_info;
     491             : 
     492          14 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_destruct\n");
     493             : 
     494          14 :         assert(raid_bdev->process == NULL);
     495             : 
     496         462 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     497             :                 /*
     498             :                  * Close all base bdev descriptors for which call has come from below
     499             :                  * layers.  Also close the descriptors if we have started shutdown.
     500             :                  */
     501         448 :                 if (g_shutdown_started || base_info->remove_scheduled == true) {
     502         448 :                         raid_bdev_free_base_bdev_resource(base_info);
     503             :                 }
     504             :         }
     505             : 
     506          14 :         if (g_shutdown_started) {
     507           0 :                 raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
     508             :         }
     509             : 
     510          14 :         if (raid_bdev->module->stop != NULL) {
     511           0 :                 if (raid_bdev->module->stop(raid_bdev) == false) {
     512           0 :                         return;
     513             :                 }
     514             :         }
     515             : 
     516          14 :         raid_bdev_module_stop_done(raid_bdev);
     517             : }
     518             : 
     519             : static int
     520          14 : raid_bdev_destruct(void *ctx)
     521             : {
     522          14 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_destruct, ctx);
     523             : 
     524          14 :         return 1;
     525             : }
     526             : 
     527             : int
     528           0 : raid_bdev_remap_dix_reftag(void *md_buf, uint64_t num_blocks,
     529             :                            struct spdk_bdev *bdev, uint32_t remapped_offset)
     530             : {
     531           0 :         struct spdk_dif_ctx dif_ctx;
     532           0 :         struct spdk_dif_error err_blk = {};
     533             :         int rc;
     534           0 :         struct spdk_dif_ctx_init_ext_opts dif_opts;
     535           0 :         struct iovec md_iov = {
     536             :                 .iov_base       = md_buf,
     537           0 :                 .iov_len        = num_blocks * bdev->md_len,
     538             :         };
     539             : 
     540           0 :         if (md_buf == NULL) {
     541           0 :                 return 0;
     542             :         }
     543             : 
     544           0 :         dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     545           0 :         dif_opts.dif_pi_format = bdev->dif_pi_format;
     546           0 :         rc = spdk_dif_ctx_init(&dif_ctx,
     547           0 :                                bdev->blocklen, bdev->md_len, bdev->md_interleave,
     548           0 :                                bdev->dif_is_head_of_md, bdev->dif_type,
     549             :                                SPDK_DIF_FLAGS_REFTAG_CHECK,
     550             :                                0, 0, 0, 0, 0, &dif_opts);
     551           0 :         if (rc != 0) {
     552           0 :                 SPDK_ERRLOG("Initialization of DIF context failed\n");
     553           0 :                 return rc;
     554             :         }
     555             : 
     556           0 :         spdk_dif_ctx_set_remapped_init_ref_tag(&dif_ctx, remapped_offset);
     557             : 
     558           0 :         rc = spdk_dix_remap_ref_tag(&md_iov, num_blocks, &dif_ctx, &err_blk, false);
     559           0 :         if (rc != 0) {
     560           0 :                 SPDK_ERRLOG("Remapping reference tag failed. type=%d, offset=%d"
     561             :                             PRIu32 "\n", err_blk.err_type, err_blk.err_offset);
     562             :         }
     563             : 
     564           0 :         return rc;
     565             : }
     566             : 
     567             : int
     568           0 : raid_bdev_verify_dix_reftag(struct iovec *iovs, int iovcnt, void *md_buf,
     569             :                             uint64_t num_blocks, struct spdk_bdev *bdev, uint32_t offset_blocks)
     570             : {
     571           0 :         struct spdk_dif_ctx dif_ctx;
     572           0 :         struct spdk_dif_error err_blk = {};
     573             :         int rc;
     574           0 :         struct spdk_dif_ctx_init_ext_opts dif_opts;
     575           0 :         struct iovec md_iov = {
     576             :                 .iov_base       = md_buf,
     577           0 :                 .iov_len        = num_blocks * bdev->md_len,
     578             :         };
     579             : 
     580           0 :         if (md_buf == NULL) {
     581           0 :                 return 0;
     582             :         }
     583             : 
     584           0 :         dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     585           0 :         dif_opts.dif_pi_format = bdev->dif_pi_format;
     586           0 :         rc = spdk_dif_ctx_init(&dif_ctx,
     587           0 :                                bdev->blocklen, bdev->md_len, bdev->md_interleave,
     588           0 :                                bdev->dif_is_head_of_md, bdev->dif_type,
     589             :                                SPDK_DIF_FLAGS_REFTAG_CHECK,
     590             :                                offset_blocks, 0, 0, 0, 0, &dif_opts);
     591           0 :         if (rc != 0) {
     592           0 :                 SPDK_ERRLOG("Initialization of DIF context failed\n");
     593           0 :                 return rc;
     594             :         }
     595             : 
     596           0 :         rc = spdk_dix_verify(iovs, iovcnt, &md_iov, num_blocks, &dif_ctx, &err_blk);
     597           0 :         if (rc != 0) {
     598           0 :                 SPDK_ERRLOG("Reference tag check failed. type=%d, offset=%d"
     599             :                             PRIu32 "\n", err_blk.err_type, err_blk.err_offset);
     600             :         }
     601             : 
     602           0 :         return rc;
     603             : }
     604             : 
     605             : void
     606          11 : raid_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status)
     607             : {
     608          11 :         struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(raid_io);
     609             :         int rc;
     610             : 
     611          11 :         spdk_trace_record(TRACE_BDEV_RAID_IO_DONE, 0, 0, (uintptr_t)raid_io, (uintptr_t)bdev_io);
     612             : 
     613          11 :         if (raid_io->split.offset != RAID_OFFSET_BLOCKS_INVALID) {
     614          10 :                 struct iovec *split_iov = raid_io->split.iov;
     615          10 :                 const struct iovec *split_iov_orig = &raid_io->split.iov_copy;
     616             : 
     617             :                 /*
     618             :                  * Non-zero offset here means that this is the completion of the first part of the
     619             :                  * split I/O (the higher LBAs). Then, we submit the second part and set offset to 0.
     620             :                  */
     621          10 :                 if (raid_io->split.offset != 0) {
     622           5 :                         raid_io->offset_blocks = bdev_io->u.bdev.offset_blocks;
     623           5 :                         raid_io->md_buf = bdev_io->u.bdev.md_buf;
     624             : 
     625           5 :                         if (status == SPDK_BDEV_IO_STATUS_SUCCESS) {
     626           5 :                                 raid_io->num_blocks = raid_io->split.offset;
     627           5 :                                 raid_io->iovcnt = raid_io->iovs - bdev_io->u.bdev.iovs;
     628           5 :                                 raid_io->iovs = bdev_io->u.bdev.iovs;
     629           5 :                                 if (split_iov != NULL) {
     630           4 :                                         raid_io->iovcnt++;
     631           4 :                                         split_iov->iov_len = split_iov->iov_base - split_iov_orig->iov_base;
     632           4 :                                         split_iov->iov_base = split_iov_orig->iov_base;
     633             :                                 }
     634             : 
     635           5 :                                 raid_io->split.offset = 0;
     636           5 :                                 raid_io->base_bdev_io_submitted = 0;
     637           5 :                                 raid_io->raid_ch = raid_io->raid_ch->process.ch_processed;
     638             : 
     639           5 :                                 raid_io->raid_bdev->module->submit_rw_request(raid_io);
     640           5 :                                 return;
     641             :                         }
     642             :                 }
     643             : 
     644           5 :                 raid_io->num_blocks = bdev_io->u.bdev.num_blocks;
     645           5 :                 raid_io->iovcnt = bdev_io->u.bdev.iovcnt;
     646           5 :                 raid_io->iovs = bdev_io->u.bdev.iovs;
     647           5 :                 if (split_iov != NULL) {
     648           4 :                         *split_iov = *split_iov_orig;
     649             :                 }
     650             :         }
     651             : 
     652           6 :         if (spdk_unlikely(raid_io->completion_cb != NULL)) {
     653           0 :                 raid_io->completion_cb(raid_io, status);
     654             :         } else {
     655           6 :                 if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
     656             :                                   spdk_bdev_get_dif_type(bdev_io->bdev) != SPDK_DIF_DISABLE &&
     657             :                                   bdev_io->bdev->dif_check_flags & SPDK_DIF_FLAGS_REFTAG_CHECK &&
     658             :                                   status == SPDK_BDEV_IO_STATUS_SUCCESS)) {
     659             : 
     660           0 :                         rc = raid_bdev_remap_dix_reftag(bdev_io->u.bdev.md_buf,
     661             :                                                         bdev_io->u.bdev.num_blocks, bdev_io->bdev,
     662           0 :                                                         bdev_io->u.bdev.offset_blocks);
     663           0 :                         if (rc != 0) {
     664           0 :                                 status = SPDK_BDEV_IO_STATUS_FAILED;
     665             :                         }
     666             :                 }
     667           6 :                 spdk_bdev_io_complete(bdev_io, status);
     668             :         }
     669             : }
     670             : 
     671             : /*
     672             :  * brief:
     673             :  * raid_bdev_io_complete_part - signal the completion of a part of the expected
     674             :  * base bdev IOs and complete the raid_io if this is the final expected IO.
     675             :  * The caller should first set raid_io->base_bdev_io_remaining. This function
     676             :  * will decrement this counter by the value of the 'completed' parameter and
     677             :  * complete the raid_io if the counter reaches 0. The caller is free to
     678             :  * interpret the 'base_bdev_io_remaining' and 'completed' values as needed,
     679             :  * it can represent e.g. blocks or IOs.
     680             :  * params:
     681             :  * raid_io - pointer to raid_bdev_io
     682             :  * completed - the part of the raid_io that has been completed
     683             :  * status - status of the base IO
     684             :  * returns:
     685             :  * true - if the raid_io is completed
     686             :  * false - otherwise
     687             :  */
     688             : bool
     689          32 : raid_bdev_io_complete_part(struct raid_bdev_io *raid_io, uint64_t completed,
     690             :                            enum spdk_bdev_io_status status)
     691             : {
     692          32 :         assert(raid_io->base_bdev_io_remaining >= completed);
     693          32 :         raid_io->base_bdev_io_remaining -= completed;
     694             : 
     695          32 :         if (status != raid_io->base_bdev_io_status_default) {
     696           0 :                 raid_io->base_bdev_io_status = status;
     697             :         }
     698             : 
     699          32 :         if (raid_io->base_bdev_io_remaining == 0) {
     700           1 :                 raid_bdev_io_complete(raid_io, raid_io->base_bdev_io_status);
     701           1 :                 return true;
     702             :         } else {
     703          31 :                 return false;
     704             :         }
     705             : }
     706             : 
     707             : /*
     708             :  * brief:
     709             :  * raid_bdev_queue_io_wait function processes the IO which failed to submit.
     710             :  * It will try to queue the IOs after storing the context to bdev wait queue logic.
     711             :  * params:
     712             :  * raid_io - pointer to raid_bdev_io
     713             :  * bdev - the block device that the IO is submitted to
     714             :  * ch - io channel
     715             :  * cb_fn - callback when the spdk_bdev_io for bdev becomes available
     716             :  * returns:
     717             :  * none
     718             :  */
     719             : void
     720           0 : raid_bdev_queue_io_wait(struct raid_bdev_io *raid_io, struct spdk_bdev *bdev,
     721             :                         struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn)
     722             : {
     723           0 :         raid_io->waitq_entry.bdev = bdev;
     724           0 :         raid_io->waitq_entry.cb_fn = cb_fn;
     725           0 :         raid_io->waitq_entry.cb_arg = raid_io;
     726           0 :         spdk_bdev_queue_io_wait(bdev, ch, &raid_io->waitq_entry);
     727           0 : }
     728             : 
     729             : static void
     730          32 : raid_base_bdev_reset_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
     731             : {
     732          32 :         struct raid_bdev_io *raid_io = cb_arg;
     733             : 
     734          32 :         spdk_bdev_free_io(bdev_io);
     735             : 
     736          32 :         raid_bdev_io_complete_part(raid_io, 1, success ?
     737             :                                    SPDK_BDEV_IO_STATUS_SUCCESS :
     738             :                                    SPDK_BDEV_IO_STATUS_FAILED);
     739          32 : }
     740             : 
     741             : static void raid_bdev_submit_reset_request(struct raid_bdev_io *raid_io);
     742             : 
     743             : static void
     744           0 : _raid_bdev_submit_reset_request(void *_raid_io)
     745             : {
     746           0 :         struct raid_bdev_io *raid_io = _raid_io;
     747             : 
     748           0 :         raid_bdev_submit_reset_request(raid_io);
     749           0 : }
     750             : 
     751             : /*
     752             :  * brief:
     753             :  * raid_bdev_submit_reset_request function submits reset requests
     754             :  * to member disks; it will submit as many as possible unless a reset fails with -ENOMEM, in
     755             :  * which case it will queue it for later submission
     756             :  * params:
     757             :  * raid_io
     758             :  * returns:
     759             :  * none
     760             :  */
     761             : static void
     762           1 : raid_bdev_submit_reset_request(struct raid_bdev_io *raid_io)
     763             : {
     764             :         struct raid_bdev                *raid_bdev;
     765             :         int                             ret;
     766             :         uint8_t                         i;
     767             :         struct raid_base_bdev_info      *base_info;
     768             :         struct spdk_io_channel          *base_ch;
     769             : 
     770           1 :         raid_bdev = raid_io->raid_bdev;
     771             : 
     772           1 :         if (raid_io->base_bdev_io_remaining == 0) {
     773           1 :                 raid_io->base_bdev_io_remaining = raid_bdev->num_base_bdevs;
     774             :         }
     775             : 
     776          33 :         for (i = raid_io->base_bdev_io_submitted; i < raid_bdev->num_base_bdevs; i++) {
     777          32 :                 base_info = &raid_bdev->base_bdev_info[i];
     778          32 :                 base_ch = raid_io->raid_ch->base_channel[i];
     779          32 :                 if (base_ch == NULL) {
     780           0 :                         raid_io->base_bdev_io_submitted++;
     781           0 :                         raid_bdev_io_complete_part(raid_io, 1, SPDK_BDEV_IO_STATUS_SUCCESS);
     782           0 :                         continue;
     783             :                 }
     784          32 :                 ret = spdk_bdev_reset(base_info->desc, base_ch,
     785             :                                       raid_base_bdev_reset_complete, raid_io);
     786          32 :                 if (ret == 0) {
     787          32 :                         raid_io->base_bdev_io_submitted++;
     788           0 :                 } else if (ret == -ENOMEM) {
     789           0 :                         raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
     790             :                                                 base_ch, _raid_bdev_submit_reset_request);
     791           0 :                         return;
     792             :                 } else {
     793           0 :                         SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
     794           0 :                         assert(false);
     795             :                         raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     796             :                         return;
     797             :                 }
     798             :         }
     799             : }
     800             : 
     801             : static void
     802           5 : raid_bdev_io_split(struct raid_bdev_io *raid_io, uint64_t split_offset)
     803             : {
     804           5 :         struct raid_bdev *raid_bdev = raid_io->raid_bdev;
     805           5 :         size_t iov_offset = split_offset * raid_bdev->bdev.blocklen;
     806             :         int i;
     807             : 
     808           5 :         assert(split_offset != 0);
     809           5 :         assert(raid_io->split.offset == RAID_OFFSET_BLOCKS_INVALID);
     810           5 :         raid_io->split.offset = split_offset;
     811             : 
     812           5 :         raid_io->offset_blocks += split_offset;
     813           5 :         raid_io->num_blocks -= split_offset;
     814           5 :         if (raid_io->md_buf != NULL) {
     815           5 :                 raid_io->md_buf += (split_offset * raid_bdev->bdev.md_len);
     816             :         }
     817             : 
     818          12 :         for (i = 0; i < raid_io->iovcnt; i++) {
     819          12 :                 struct iovec *iov = &raid_io->iovs[i];
     820             : 
     821          12 :                 if (iov_offset < iov->iov_len) {
     822           5 :                         if (iov_offset == 0) {
     823           1 :                                 raid_io->split.iov = NULL;
     824             :                         } else {
     825           4 :                                 raid_io->split.iov = iov;
     826           4 :                                 raid_io->split.iov_copy = *iov;
     827           4 :                                 iov->iov_base += iov_offset;
     828           4 :                                 iov->iov_len -= iov_offset;
     829             :                         }
     830           5 :                         raid_io->iovs += i;
     831           5 :                         raid_io->iovcnt -= i;
     832           5 :                         break;
     833             :                 }
     834             : 
     835           7 :                 iov_offset -= iov->iov_len;
     836             :         }
     837           5 : }
     838             : 
     839             : static void
     840           5 : raid_bdev_submit_rw_request(struct raid_bdev_io *raid_io)
     841             : {
     842           5 :         struct raid_bdev_io_channel *raid_ch = raid_io->raid_ch;
     843             : 
     844           5 :         if (raid_ch->process.offset != RAID_OFFSET_BLOCKS_INVALID) {
     845           5 :                 uint64_t offset_begin = raid_io->offset_blocks;
     846           5 :                 uint64_t offset_end = offset_begin + raid_io->num_blocks;
     847             : 
     848           5 :                 if (offset_end > raid_ch->process.offset) {
     849           5 :                         if (offset_begin < raid_ch->process.offset) {
     850             :                                 /*
     851             :                                  * If the I/O spans both the processed and unprocessed ranges,
     852             :                                  * split it and first handle the unprocessed part. After it
     853             :                                  * completes, the rest will be handled.
     854             :                                  * This situation occurs when the process thread is not active
     855             :                                  * or is waiting for the process window range to be locked
     856             :                                  * (quiesced). When a window is being processed, such I/Os will be
     857             :                                  * deferred by the bdev layer until the window is unlocked.
     858             :                                  */
     859           5 :                                 SPDK_DEBUGLOG(bdev_raid, "split: process_offset: %lu offset_begin: %lu offset_end: %lu\n",
     860             :                                               raid_ch->process.offset, offset_begin, offset_end);
     861           5 :                                 raid_bdev_io_split(raid_io, raid_ch->process.offset - offset_begin);
     862             :                         }
     863             :                 } else {
     864             :                         /* Use the child channel, which corresponds to the already processed range */
     865           0 :                         raid_io->raid_ch = raid_ch->process.ch_processed;
     866             :                 }
     867             :         }
     868             : 
     869           5 :         raid_io->raid_bdev->module->submit_rw_request(raid_io);
     870           5 : }
     871             : 
     872             : /*
     873             :  * brief:
     874             :  * Callback function to spdk_bdev_io_get_buf.
     875             :  * params:
     876             :  * ch - pointer to raid bdev io channel
     877             :  * bdev_io - pointer to parent bdev_io on raid bdev device
     878             :  * success - True if buffer is allocated or false otherwise.
     879             :  * returns:
     880             :  * none
     881             :  */
     882             : static void
     883           0 : raid_bdev_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
     884             :                      bool success)
     885             : {
     886           0 :         struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx;
     887             : 
     888           0 :         if (!success) {
     889           0 :                 raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     890           0 :                 return;
     891             :         }
     892             : 
     893           0 :         raid_bdev_submit_rw_request(raid_io);
     894             : }
     895             : 
     896             : void
     897           6 : raid_bdev_io_init(struct raid_bdev_io *raid_io, struct raid_bdev_io_channel *raid_ch,
     898             :                   enum spdk_bdev_io_type type, uint64_t offset_blocks,
     899             :                   uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf,
     900             :                   struct spdk_memory_domain *memory_domain, void *memory_domain_ctx)
     901             : {
     902           6 :         struct spdk_io_channel *ch = spdk_io_channel_from_ctx(raid_ch);
     903           6 :         struct raid_bdev *raid_bdev = spdk_io_channel_get_io_device(ch);
     904             : 
     905           6 :         raid_io->type = type;
     906           6 :         raid_io->offset_blocks = offset_blocks;
     907           6 :         raid_io->num_blocks = num_blocks;
     908           6 :         raid_io->iovs = iovs;
     909           6 :         raid_io->iovcnt = iovcnt;
     910           6 :         raid_io->memory_domain = memory_domain;
     911           6 :         raid_io->memory_domain_ctx = memory_domain_ctx;
     912           6 :         raid_io->md_buf = md_buf;
     913             : 
     914           6 :         raid_io->raid_bdev = raid_bdev;
     915           6 :         raid_io->raid_ch = raid_ch;
     916           6 :         raid_io->base_bdev_io_remaining = 0;
     917           6 :         raid_io->base_bdev_io_submitted = 0;
     918           6 :         raid_io->completion_cb = NULL;
     919           6 :         raid_io->split.offset = RAID_OFFSET_BLOCKS_INVALID;
     920             : 
     921           6 :         raid_bdev_io_set_default_status(raid_io, SPDK_BDEV_IO_STATUS_SUCCESS);
     922           6 : }
     923             : 
     924             : /*
     925             :  * brief:
     926             :  * raid_bdev_submit_request function is the submit_request function pointer of
     927             :  * raid bdev function table. This is used to submit the io on raid_bdev to below
     928             :  * layers.
     929             :  * params:
     930             :  * ch - pointer to raid bdev io channel
     931             :  * bdev_io - pointer to parent bdev_io on raid bdev device
     932             :  * returns:
     933             :  * none
     934             :  */
     935             : static void
     936           6 : raid_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
     937             : {
     938           6 :         struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx;
     939             : 
     940           6 :         raid_bdev_io_init(raid_io, spdk_io_channel_get_ctx(ch), bdev_io->type,
     941             :                           bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
     942             :                           bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.md_buf,
     943             :                           bdev_io->u.bdev.memory_domain, bdev_io->u.bdev.memory_domain_ctx);
     944             : 
     945           6 :         spdk_trace_record(TRACE_BDEV_RAID_IO_START, 0, 0, (uintptr_t)raid_io, (uintptr_t)bdev_io);
     946             : 
     947           6 :         switch (bdev_io->type) {
     948           0 :         case SPDK_BDEV_IO_TYPE_READ:
     949           0 :                 spdk_bdev_io_get_buf(bdev_io, raid_bdev_get_buf_cb,
     950           0 :                                      bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
     951           0 :                 break;
     952           5 :         case SPDK_BDEV_IO_TYPE_WRITE:
     953           5 :                 raid_bdev_submit_rw_request(raid_io);
     954           5 :                 break;
     955             : 
     956           1 :         case SPDK_BDEV_IO_TYPE_RESET:
     957           1 :                 raid_bdev_submit_reset_request(raid_io);
     958           1 :                 break;
     959             : 
     960           0 :         case SPDK_BDEV_IO_TYPE_FLUSH:
     961             :         case SPDK_BDEV_IO_TYPE_UNMAP:
     962           0 :                 if (raid_io->raid_bdev->process != NULL) {
     963             :                         /* TODO: rebuild support */
     964           0 :                         raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     965           0 :                         return;
     966             :                 }
     967           0 :                 raid_io->raid_bdev->module->submit_null_payload_request(raid_io);
     968           0 :                 break;
     969             : 
     970           0 :         default:
     971           0 :                 SPDK_ERRLOG("submit request, invalid io type %u\n", bdev_io->type);
     972           0 :                 raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     973           0 :                 break;
     974             :         }
     975             : }
     976             : 
     977             : /*
     978             :  * brief:
     979             :  * _raid_bdev_io_type_supported checks whether io_type is supported in
     980             :  * all base bdev modules of raid bdev module. If anyone among the base_bdevs
     981             :  * doesn't support, the raid device doesn't supports.
     982             :  *
     983             :  * params:
     984             :  * raid_bdev - pointer to raid bdev context
     985             :  * io_type - io type
     986             :  * returns:
     987             :  * true - io_type is supported
     988             :  * false - io_type is not supported
     989             :  */
     990             : inline static bool
     991           1 : _raid_bdev_io_type_supported(struct raid_bdev *raid_bdev, enum spdk_bdev_io_type io_type)
     992             : {
     993             :         struct raid_base_bdev_info *base_info;
     994             : 
     995           1 :         if (io_type == SPDK_BDEV_IO_TYPE_FLUSH ||
     996             :             io_type == SPDK_BDEV_IO_TYPE_UNMAP) {
     997           0 :                 if (raid_bdev->module->submit_null_payload_request == NULL) {
     998           0 :                         return false;
     999             :                 }
    1000             :         }
    1001             : 
    1002          33 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1003          32 :                 if (base_info->desc == NULL) {
    1004           0 :                         continue;
    1005             :                 }
    1006             : 
    1007          32 :                 if (spdk_bdev_io_type_supported(spdk_bdev_desc_get_bdev(base_info->desc), io_type) == false) {
    1008           0 :                         return false;
    1009             :                 }
    1010             :         }
    1011             : 
    1012           1 :         return true;
    1013             : }
    1014             : 
    1015             : /*
    1016             :  * brief:
    1017             :  * raid_bdev_io_type_supported is the io_supported function for bdev function
    1018             :  * table which returns whether the particular io type is supported or not by
    1019             :  * raid bdev module
    1020             :  * params:
    1021             :  * ctx - pointer to raid bdev context
    1022             :  * type - io type
    1023             :  * returns:
    1024             :  * true - io_type is supported
    1025             :  * false - io_type is not supported
    1026             :  */
    1027             : static bool
    1028           4 : raid_bdev_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
    1029             : {
    1030           4 :         switch (io_type) {
    1031           2 :         case SPDK_BDEV_IO_TYPE_READ:
    1032             :         case SPDK_BDEV_IO_TYPE_WRITE:
    1033           2 :                 return true;
    1034             : 
    1035           1 :         case SPDK_BDEV_IO_TYPE_FLUSH:
    1036             :         case SPDK_BDEV_IO_TYPE_RESET:
    1037             :         case SPDK_BDEV_IO_TYPE_UNMAP:
    1038           1 :                 return _raid_bdev_io_type_supported(ctx, io_type);
    1039             : 
    1040           1 :         default:
    1041           1 :                 return false;
    1042             :         }
    1043             : 
    1044             :         return false;
    1045             : }
    1046             : 
    1047             : /*
    1048             :  * brief:
    1049             :  * raid_bdev_get_io_channel is the get_io_channel function table pointer for
    1050             :  * raid bdev. This is used to return the io channel for this raid bdev
    1051             :  * params:
    1052             :  * ctxt - pointer to raid_bdev
    1053             :  * returns:
    1054             :  * pointer to io channel for raid bdev
    1055             :  */
    1056             : static struct spdk_io_channel *
    1057           0 : raid_bdev_get_io_channel(void *ctxt)
    1058             : {
    1059           0 :         struct raid_bdev *raid_bdev = ctxt;
    1060             : 
    1061           0 :         return spdk_get_io_channel(raid_bdev);
    1062             : }
    1063             : 
    1064             : void
    1065           7 : raid_bdev_write_info_json(struct raid_bdev *raid_bdev, struct spdk_json_write_ctx *w)
    1066             : {
    1067             :         struct raid_base_bdev_info *base_info;
    1068             : 
    1069           7 :         assert(raid_bdev != NULL);
    1070           7 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    1071             : 
    1072           7 :         spdk_json_write_named_uuid(w, "uuid", &raid_bdev->bdev.uuid);
    1073           7 :         spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
    1074           7 :         spdk_json_write_named_string(w, "state", raid_bdev_state_to_str(raid_bdev->state));
    1075           7 :         spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
    1076           7 :         spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
    1077           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs", raid_bdev->num_base_bdevs);
    1078           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", raid_bdev->num_base_bdevs_discovered);
    1079           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs_operational",
    1080           7 :                                      raid_bdev->num_base_bdevs_operational);
    1081           7 :         if (raid_bdev->process) {
    1082           0 :                 struct raid_bdev_process *process = raid_bdev->process;
    1083           0 :                 uint64_t offset = process->window_offset;
    1084             : 
    1085           0 :                 spdk_json_write_named_object_begin(w, "process");
    1086           0 :                 spdk_json_write_name(w, "type");
    1087           0 :                 spdk_json_write_string(w, raid_bdev_process_to_str(process->type));
    1088           0 :                 spdk_json_write_named_string(w, "target", process->target->name);
    1089           0 :                 spdk_json_write_named_object_begin(w, "progress");
    1090           0 :                 spdk_json_write_named_uint64(w, "blocks", offset);
    1091           0 :                 spdk_json_write_named_uint32(w, "percent", offset * 100.0 / raid_bdev->bdev.blockcnt);
    1092           0 :                 spdk_json_write_object_end(w);
    1093           0 :                 spdk_json_write_object_end(w);
    1094             :         }
    1095           7 :         spdk_json_write_name(w, "base_bdevs_list");
    1096           7 :         spdk_json_write_array_begin(w);
    1097         231 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1098         224 :                 spdk_json_write_object_begin(w);
    1099         224 :                 spdk_json_write_name(w, "name");
    1100         224 :                 if (base_info->name) {
    1101         224 :                         spdk_json_write_string(w, base_info->name);
    1102             :                 } else {
    1103           0 :                         spdk_json_write_null(w);
    1104             :                 }
    1105         224 :                 spdk_json_write_named_uuid(w, "uuid", &base_info->uuid);
    1106         224 :                 spdk_json_write_named_bool(w, "is_configured", base_info->is_configured);
    1107         224 :                 spdk_json_write_named_uint64(w, "data_offset", base_info->data_offset);
    1108         224 :                 spdk_json_write_named_uint64(w, "data_size", base_info->data_size);
    1109         224 :                 spdk_json_write_object_end(w);
    1110             :         }
    1111           7 :         spdk_json_write_array_end(w);
    1112           7 : }
    1113             : 
    1114             : /*
    1115             :  * brief:
    1116             :  * raid_bdev_dump_info_json is the function table pointer for raid bdev
    1117             :  * params:
    1118             :  * ctx - pointer to raid_bdev
    1119             :  * w - pointer to json context
    1120             :  * returns:
    1121             :  * 0 - success
    1122             :  * non zero - failure
    1123             :  */
    1124             : static int
    1125           1 : raid_bdev_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
    1126             : {
    1127           1 :         struct raid_bdev *raid_bdev = ctx;
    1128             : 
    1129           1 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_dump_config_json\n");
    1130             : 
    1131             :         /* Dump the raid bdev configuration related information */
    1132           1 :         spdk_json_write_named_object_begin(w, "raid");
    1133           1 :         raid_bdev_write_info_json(raid_bdev, w);
    1134           1 :         spdk_json_write_object_end(w);
    1135             : 
    1136           1 :         return 0;
    1137             : }
    1138             : 
    1139             : /*
    1140             :  * brief:
    1141             :  * raid_bdev_write_config_json is the function table pointer for raid bdev
    1142             :  * params:
    1143             :  * bdev - pointer to spdk_bdev
    1144             :  * w - pointer to json context
    1145             :  * returns:
    1146             :  * none
    1147             :  */
    1148             : static void
    1149           0 : raid_bdev_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
    1150             : {
    1151           0 :         struct raid_bdev *raid_bdev = bdev->ctxt;
    1152             :         struct raid_base_bdev_info *base_info;
    1153             : 
    1154           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    1155             : 
    1156           0 :         if (raid_bdev->superblock_enabled) {
    1157             :                 /* raid bdev configuration is stored in the superblock */
    1158           0 :                 return;
    1159             :         }
    1160             : 
    1161           0 :         spdk_json_write_object_begin(w);
    1162             : 
    1163           0 :         spdk_json_write_named_string(w, "method", "bdev_raid_create");
    1164             : 
    1165           0 :         spdk_json_write_named_object_begin(w, "params");
    1166           0 :         spdk_json_write_named_string(w, "name", bdev->name);
    1167           0 :         spdk_json_write_named_uuid(w, "uuid", &raid_bdev->bdev.uuid);
    1168           0 :         if (raid_bdev->strip_size_kb != 0) {
    1169           0 :                 spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
    1170             :         }
    1171           0 :         spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
    1172             : 
    1173           0 :         spdk_json_write_named_array_begin(w, "base_bdevs");
    1174           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1175           0 :                 if (base_info->name) {
    1176           0 :                         spdk_json_write_string(w, base_info->name);
    1177             :                 } else {
    1178           0 :                         char str[32];
    1179             : 
    1180           0 :                         snprintf(str, sizeof(str), "removed_base_bdev_%u", raid_bdev_base_bdev_slot(base_info));
    1181           0 :                         spdk_json_write_string(w, str);
    1182             :                 }
    1183             :         }
    1184           0 :         spdk_json_write_array_end(w);
    1185           0 :         spdk_json_write_object_end(w);
    1186             : 
    1187           0 :         spdk_json_write_object_end(w);
    1188             : }
    1189             : 
    1190             : static int
    1191           0 : raid_bdev_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
    1192             : {
    1193           0 :         struct raid_bdev *raid_bdev = ctx;
    1194             :         struct raid_base_bdev_info *base_info;
    1195           0 :         int domains_count = 0, rc = 0;
    1196             : 
    1197           0 :         if (raid_bdev->module->memory_domains_supported == false) {
    1198           0 :                 return 0;
    1199             :         }
    1200             : 
    1201             :         /* First loop to get the number of memory domains */
    1202           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1203           0 :                 if (base_info->is_configured == false) {
    1204           0 :                         continue;
    1205             :                 }
    1206           0 :                 rc = spdk_bdev_get_memory_domains(spdk_bdev_desc_get_bdev(base_info->desc), NULL, 0);
    1207           0 :                 if (rc < 0) {
    1208           0 :                         return rc;
    1209             :                 }
    1210           0 :                 domains_count += rc;
    1211             :         }
    1212             : 
    1213           0 :         if (!domains || array_size < domains_count) {
    1214           0 :                 return domains_count;
    1215             :         }
    1216             : 
    1217           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1218           0 :                 if (base_info->is_configured == false) {
    1219           0 :                         continue;
    1220             :                 }
    1221           0 :                 rc = spdk_bdev_get_memory_domains(spdk_bdev_desc_get_bdev(base_info->desc), domains, array_size);
    1222           0 :                 if (rc < 0) {
    1223           0 :                         return rc;
    1224             :                 }
    1225           0 :                 domains += rc;
    1226           0 :                 array_size -= rc;
    1227             :         }
    1228             : 
    1229           0 :         return domains_count;
    1230             : }
    1231             : 
    1232             : /* g_raid_bdev_fn_table is the function table for raid bdev */
    1233             : static const struct spdk_bdev_fn_table g_raid_bdev_fn_table = {
    1234             :         .destruct               = raid_bdev_destruct,
    1235             :         .submit_request         = raid_bdev_submit_request,
    1236             :         .io_type_supported      = raid_bdev_io_type_supported,
    1237             :         .get_io_channel         = raid_bdev_get_io_channel,
    1238             :         .dump_info_json         = raid_bdev_dump_info_json,
    1239             :         .write_config_json      = raid_bdev_write_config_json,
    1240             :         .get_memory_domains     = raid_bdev_get_memory_domains,
    1241             : };
    1242             : 
    1243             : struct raid_bdev *
    1244          36 : raid_bdev_find_by_name(const char *name)
    1245             : {
    1246             :         struct raid_bdev *raid_bdev;
    1247             : 
    1248          43 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1249          23 :                 if (strcmp(raid_bdev->bdev.name, name) == 0) {
    1250          16 :                         return raid_bdev;
    1251             :                 }
    1252             :         }
    1253             : 
    1254          20 :         return NULL;
    1255             : }
    1256             : 
    1257             : static struct raid_bdev *
    1258           0 : raid_bdev_find_by_uuid(const struct spdk_uuid *uuid)
    1259             : {
    1260             :         struct raid_bdev *raid_bdev;
    1261             : 
    1262           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1263           0 :                 if (spdk_uuid_compare(&raid_bdev->bdev.uuid, uuid) == 0) {
    1264           0 :                         return raid_bdev;
    1265             :                 }
    1266             :         }
    1267             : 
    1268           0 :         return NULL;
    1269             : }
    1270             : 
    1271             : static struct {
    1272             :         const char *name;
    1273             :         enum raid_level value;
    1274             : } g_raid_level_names[] = {
    1275             :         { "raid0", RAID0 },
    1276             :         { "0", RAID0 },
    1277             :         { "raid1", RAID1 },
    1278             :         { "1", RAID1 },
    1279             :         { "raid5f", RAID5F },
    1280             :         { "5f", RAID5F },
    1281             :         { "concat", CONCAT },
    1282             :         { }
    1283             : };
    1284             : 
    1285             : const char *g_raid_state_names[] = {
    1286             :         [RAID_BDEV_STATE_ONLINE]        = "online",
    1287             :         [RAID_BDEV_STATE_CONFIGURING]   = "configuring",
    1288             :         [RAID_BDEV_STATE_OFFLINE]       = "offline",
    1289             :         [RAID_BDEV_STATE_MAX]           = NULL
    1290             : };
    1291             : 
    1292             : static const char *g_raid_process_type_names[] = {
    1293             :         [RAID_PROCESS_NONE]     = "none",
    1294             :         [RAID_PROCESS_REBUILD]  = "rebuild",
    1295             :         [RAID_PROCESS_MAX]      = NULL
    1296             : };
    1297             : 
    1298             : /* We have to use the typedef in the function declaration to appease astyle. */
    1299             : typedef enum raid_level raid_level_t;
    1300             : typedef enum raid_bdev_state raid_bdev_state_t;
    1301             : 
    1302             : raid_level_t
    1303           4 : raid_bdev_str_to_level(const char *str)
    1304             : {
    1305             :         unsigned int i;
    1306             : 
    1307           4 :         assert(str != NULL);
    1308             : 
    1309          12 :         for (i = 0; g_raid_level_names[i].name != NULL; i++) {
    1310          11 :                 if (strcasecmp(g_raid_level_names[i].name, str) == 0) {
    1311           3 :                         return g_raid_level_names[i].value;
    1312             :                 }
    1313             :         }
    1314             : 
    1315           1 :         return INVALID_RAID_LEVEL;
    1316             : }
    1317             : 
    1318             : const char *
    1319          11 : raid_bdev_level_to_str(enum raid_level level)
    1320             : {
    1321             :         unsigned int i;
    1322             : 
    1323          81 :         for (i = 0; g_raid_level_names[i].name != NULL; i++) {
    1324          71 :                 if (g_raid_level_names[i].value == level) {
    1325           1 :                         return g_raid_level_names[i].name;
    1326             :                 }
    1327             :         }
    1328             : 
    1329          10 :         return "";
    1330             : }
    1331             : 
    1332             : raid_bdev_state_t
    1333           6 : raid_bdev_str_to_state(const char *str)
    1334             : {
    1335             :         unsigned int i;
    1336             : 
    1337           6 :         assert(str != NULL);
    1338             : 
    1339          18 :         for (i = 0; i < RAID_BDEV_STATE_MAX; i++) {
    1340          15 :                 if (strcasecmp(g_raid_state_names[i], str) == 0) {
    1341           3 :                         break;
    1342             :                 }
    1343             :         }
    1344             : 
    1345           6 :         return i;
    1346             : }
    1347             : 
    1348             : const char *
    1349           7 : raid_bdev_state_to_str(enum raid_bdev_state state)
    1350             : {
    1351           7 :         if (state >= RAID_BDEV_STATE_MAX) {
    1352           0 :                 return "";
    1353             :         }
    1354             : 
    1355           7 :         return g_raid_state_names[state];
    1356             : }
    1357             : 
    1358             : const char *
    1359           6 : raid_bdev_process_to_str(enum raid_process_type value)
    1360             : {
    1361           6 :         if (value >= RAID_PROCESS_MAX) {
    1362           0 :                 return "";
    1363             :         }
    1364             : 
    1365           6 :         return g_raid_process_type_names[value];
    1366             : }
    1367             : 
    1368             : /*
    1369             :  * brief:
    1370             :  * raid_bdev_fini_start is called when bdev layer is starting the
    1371             :  * shutdown process
    1372             :  * params:
    1373             :  * none
    1374             :  * returns:
    1375             :  * none
    1376             :  */
    1377             : static void
    1378           0 : raid_bdev_fini_start(void)
    1379             : {
    1380             :         struct raid_bdev *raid_bdev;
    1381             :         struct raid_base_bdev_info *base_info;
    1382             : 
    1383           0 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_fini_start\n");
    1384             : 
    1385           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1386           0 :                 if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    1387           0 :                         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1388           0 :                                 raid_bdev_free_base_bdev_resource(base_info);
    1389             :                         }
    1390             :                 }
    1391             :         }
    1392             : 
    1393           0 :         g_shutdown_started = true;
    1394           0 : }
    1395             : 
    1396             : /*
    1397             :  * brief:
    1398             :  * raid_bdev_exit is called on raid bdev module exit time by bdev layer
    1399             :  * params:
    1400             :  * none
    1401             :  * returns:
    1402             :  * none
    1403             :  */
    1404             : static void
    1405          12 : raid_bdev_exit(void)
    1406             : {
    1407             :         struct raid_bdev *raid_bdev, *tmp;
    1408             : 
    1409          12 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_exit\n");
    1410             : 
    1411          12 :         TAILQ_FOREACH_SAFE(raid_bdev, &g_raid_bdev_list, global_link, tmp) {
    1412           0 :                 raid_bdev_cleanup_and_free(raid_bdev);
    1413             :         }
    1414          12 : }
    1415             : 
    1416             : static void
    1417           0 : raid_bdev_opts_config_json(struct spdk_json_write_ctx *w)
    1418             : {
    1419           0 :         spdk_json_write_object_begin(w);
    1420             : 
    1421           0 :         spdk_json_write_named_string(w, "method", "bdev_raid_set_options");
    1422             : 
    1423           0 :         spdk_json_write_named_object_begin(w, "params");
    1424           0 :         spdk_json_write_named_uint32(w, "process_window_size_kb", g_opts.process_window_size_kb);
    1425           0 :         spdk_json_write_named_uint32(w, "process_max_bandwidth_mb_sec",
    1426             :                                      g_opts.process_max_bandwidth_mb_sec);
    1427           0 :         spdk_json_write_object_end(w);
    1428             : 
    1429           0 :         spdk_json_write_object_end(w);
    1430           0 : }
    1431             : 
    1432             : static int
    1433           0 : raid_bdev_config_json(struct spdk_json_write_ctx *w)
    1434             : {
    1435           0 :         raid_bdev_opts_config_json(w);
    1436             : 
    1437           0 :         return 0;
    1438             : }
    1439             : 
    1440             : /*
    1441             :  * brief:
    1442             :  * raid_bdev_get_ctx_size is used to return the context size of bdev_io for raid
    1443             :  * module
    1444             :  * params:
    1445             :  * none
    1446             :  * returns:
    1447             :  * size of spdk_bdev_io context for raid
    1448             :  */
    1449             : static int
    1450           1 : raid_bdev_get_ctx_size(void)
    1451             : {
    1452           1 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_get_ctx_size\n");
    1453           1 :         return sizeof(struct raid_bdev_io);
    1454             : }
    1455             : 
    1456             : static struct spdk_bdev_module g_raid_if = {
    1457             :         .name = "raid",
    1458             :         .module_init = raid_bdev_init,
    1459             :         .fini_start = raid_bdev_fini_start,
    1460             :         .module_fini = raid_bdev_exit,
    1461             :         .config_json = raid_bdev_config_json,
    1462             :         .get_ctx_size = raid_bdev_get_ctx_size,
    1463             :         .examine_disk = raid_bdev_examine,
    1464             :         .async_init = false,
    1465             :         .async_fini = false,
    1466             : };
    1467           1 : SPDK_BDEV_MODULE_REGISTER(raid, &g_raid_if)
    1468             : 
    1469             : /*
    1470             :  * brief:
    1471             :  * raid_bdev_init is the initialization function for raid bdev module
    1472             :  * params:
    1473             :  * none
    1474             :  * returns:
    1475             :  * 0 - success
    1476             :  * non zero - failure
    1477             :  */
    1478             : static int
    1479          12 : raid_bdev_init(void)
    1480             : {
    1481          12 :         return 0;
    1482             : }
    1483             : 
    1484             : static int
    1485          20 : _raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
    1486             :                   enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
    1487             :                   struct raid_bdev **raid_bdev_out)
    1488             : {
    1489             :         struct raid_bdev *raid_bdev;
    1490             :         struct spdk_bdev *raid_bdev_gen;
    1491             :         struct raid_bdev_module *module;
    1492             :         struct raid_base_bdev_info *base_info;
    1493             :         uint8_t min_operational;
    1494             : 
    1495          20 :         if (strnlen(name, RAID_BDEV_SB_NAME_SIZE) == RAID_BDEV_SB_NAME_SIZE) {
    1496           0 :                 SPDK_ERRLOG("Raid bdev name '%s' exceeds %d characters\n", name, RAID_BDEV_SB_NAME_SIZE - 1);
    1497           0 :                 return -EINVAL;
    1498             :         }
    1499             : 
    1500          20 :         if (raid_bdev_find_by_name(name) != NULL) {
    1501           1 :                 SPDK_ERRLOG("Duplicate raid bdev name found: %s\n", name);
    1502           1 :                 return -EEXIST;
    1503             :         }
    1504             : 
    1505          19 :         if (level == RAID1) {
    1506           0 :                 if (strip_size != 0) {
    1507           0 :                         SPDK_ERRLOG("Strip size is not supported by raid1\n");
    1508           0 :                         return -EINVAL;
    1509             :                 }
    1510          19 :         } else if (spdk_u32_is_pow2(strip_size) == false) {
    1511           1 :                 SPDK_ERRLOG("Invalid strip size %" PRIu32 "\n", strip_size);
    1512           1 :                 return -EINVAL;
    1513             :         }
    1514             : 
    1515          18 :         module = raid_bdev_module_find(level);
    1516          18 :         if (module == NULL) {
    1517           1 :                 SPDK_ERRLOG("Unsupported raid level '%d'\n", level);
    1518           1 :                 return -EINVAL;
    1519             :         }
    1520             : 
    1521          17 :         assert(module->base_bdevs_min != 0);
    1522          17 :         if (num_base_bdevs < module->base_bdevs_min) {
    1523           0 :                 SPDK_ERRLOG("At least %u base devices required for %s\n",
    1524             :                             module->base_bdevs_min,
    1525             :                             raid_bdev_level_to_str(level));
    1526           0 :                 return -EINVAL;
    1527             :         }
    1528             : 
    1529          17 :         switch (module->base_bdevs_constraint.type) {
    1530           0 :         case CONSTRAINT_MAX_BASE_BDEVS_REMOVED:
    1531           0 :                 min_operational = num_base_bdevs - module->base_bdevs_constraint.value;
    1532           0 :                 break;
    1533           0 :         case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL:
    1534           0 :                 min_operational = module->base_bdevs_constraint.value;
    1535           0 :                 break;
    1536          17 :         case CONSTRAINT_UNSET:
    1537          17 :                 if (module->base_bdevs_constraint.value != 0) {
    1538           0 :                         SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid bdev '%s'.\n",
    1539             :                                     (uint8_t)module->base_bdevs_constraint.value, name);
    1540           0 :                         return -EINVAL;
    1541             :                 }
    1542          17 :                 min_operational = num_base_bdevs;
    1543          17 :                 break;
    1544           0 :         default:
    1545           0 :                 SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid level '%s'.\n",
    1546             :                             (uint8_t)module->base_bdevs_constraint.type,
    1547             :                             raid_bdev_level_to_str(module->level));
    1548           0 :                 return -EINVAL;
    1549             :         };
    1550             : 
    1551          17 :         if (min_operational == 0 || min_operational > num_base_bdevs) {
    1552           0 :                 SPDK_ERRLOG("Wrong constraint value for raid level '%s'.\n",
    1553             :                             raid_bdev_level_to_str(module->level));
    1554           0 :                 return -EINVAL;
    1555             :         }
    1556             : 
    1557          17 :         raid_bdev = calloc(1, sizeof(*raid_bdev));
    1558          17 :         if (!raid_bdev) {
    1559           0 :                 SPDK_ERRLOG("Unable to allocate memory for raid bdev\n");
    1560           0 :                 return -ENOMEM;
    1561             :         }
    1562             : 
    1563          17 :         raid_bdev->module = module;
    1564          17 :         raid_bdev->num_base_bdevs = num_base_bdevs;
    1565          17 :         raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs,
    1566             :                                            sizeof(struct raid_base_bdev_info));
    1567          17 :         if (!raid_bdev->base_bdev_info) {
    1568           0 :                 SPDK_ERRLOG("Unable able to allocate base bdev info\n");
    1569           0 :                 raid_bdev_free(raid_bdev);
    1570           0 :                 return -ENOMEM;
    1571             :         }
    1572             : 
    1573         561 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1574         544 :                 base_info->raid_bdev = raid_bdev;
    1575             :         }
    1576             : 
    1577             :         /* strip_size_kb is from the rpc param.  strip_size is in blocks and used
    1578             :          * internally and set later.
    1579             :          */
    1580          17 :         raid_bdev->strip_size = 0;
    1581          17 :         raid_bdev->strip_size_kb = strip_size;
    1582          17 :         raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
    1583          17 :         raid_bdev->level = level;
    1584          17 :         raid_bdev->min_base_bdevs_operational = min_operational;
    1585          17 :         raid_bdev->superblock_enabled = superblock_enabled;
    1586             : 
    1587          17 :         raid_bdev_gen = &raid_bdev->bdev;
    1588             : 
    1589          17 :         raid_bdev_gen->name = strdup(name);
    1590          17 :         if (!raid_bdev_gen->name) {
    1591           0 :                 SPDK_ERRLOG("Unable to allocate name for raid\n");
    1592           0 :                 raid_bdev_free(raid_bdev);
    1593           0 :                 return -ENOMEM;
    1594             :         }
    1595             : 
    1596          17 :         raid_bdev_gen->product_name = "Raid Volume";
    1597          17 :         raid_bdev_gen->ctxt = raid_bdev;
    1598          17 :         raid_bdev_gen->fn_table = &g_raid_bdev_fn_table;
    1599          17 :         raid_bdev_gen->module = &g_raid_if;
    1600          17 :         raid_bdev_gen->write_cache = 0;
    1601          17 :         spdk_uuid_copy(&raid_bdev_gen->uuid, uuid);
    1602             : 
    1603          17 :         TAILQ_INSERT_TAIL(&g_raid_bdev_list, raid_bdev, global_link);
    1604             : 
    1605          17 :         *raid_bdev_out = raid_bdev;
    1606             : 
    1607          17 :         return 0;
    1608             : }
    1609             : 
    1610             : /*
    1611             :  * brief:
    1612             :  * raid_bdev_create allocates raid bdev based on passed configuration
    1613             :  * params:
    1614             :  * name - name for raid bdev
    1615             :  * strip_size - strip size in KB
    1616             :  * num_base_bdevs - number of base bdevs
    1617             :  * level - raid level
    1618             :  * superblock_enabled - true if raid should have superblock
    1619             :  * uuid - uuid to set for the bdev
    1620             :  * raid_bdev_out - the created raid bdev
    1621             :  * returns:
    1622             :  * 0 - success
    1623             :  * non zero - failure
    1624             :  */
    1625             : int
    1626          20 : raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
    1627             :                  enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
    1628             :                  struct raid_bdev **raid_bdev_out)
    1629             : {
    1630          20 :         struct raid_bdev *raid_bdev;
    1631             :         int rc;
    1632             : 
    1633          20 :         assert(uuid != NULL);
    1634             : 
    1635          20 :         rc = _raid_bdev_create(name, strip_size, num_base_bdevs, level, superblock_enabled, uuid,
    1636             :                                &raid_bdev);
    1637          20 :         if (rc != 0) {
    1638           3 :                 return rc;
    1639             :         }
    1640             : 
    1641          17 :         if (superblock_enabled && spdk_uuid_is_null(uuid)) {
    1642             :                 /* we need to have the uuid to store in the superblock before the bdev is registered */
    1643           1 :                 spdk_uuid_generate(&raid_bdev->bdev.uuid);
    1644             :         }
    1645             : 
    1646          17 :         raid_bdev->num_base_bdevs_operational = num_base_bdevs;
    1647             : 
    1648          17 :         *raid_bdev_out = raid_bdev;
    1649             : 
    1650          17 :         return 0;
    1651             : }
    1652             : 
    1653             : static void
    1654           0 : _raid_bdev_unregistering_cont(void *ctx)
    1655             : {
    1656           0 :         struct raid_bdev *raid_bdev = ctx;
    1657             : 
    1658           0 :         spdk_bdev_close(raid_bdev->self_desc);
    1659           0 :         raid_bdev->self_desc = NULL;
    1660           0 : }
    1661             : 
    1662             : static void
    1663           0 : raid_bdev_unregistering_cont(void *ctx)
    1664             : {
    1665           0 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_unregistering_cont, ctx);
    1666           0 : }
    1667             : 
    1668             : static int
    1669           0 : raid_bdev_process_add_finish_action(struct raid_bdev_process *process, spdk_msg_fn cb, void *cb_ctx)
    1670             : {
    1671             :         struct raid_process_finish_action *finish_action;
    1672             : 
    1673           0 :         assert(spdk_get_thread() == process->thread);
    1674           0 :         assert(process->state < RAID_PROCESS_STATE_STOPPED);
    1675             : 
    1676           0 :         finish_action = calloc(1, sizeof(*finish_action));
    1677           0 :         if (finish_action == NULL) {
    1678           0 :                 return -ENOMEM;
    1679             :         }
    1680             : 
    1681           0 :         finish_action->cb = cb;
    1682           0 :         finish_action->cb_ctx = cb_ctx;
    1683             : 
    1684           0 :         TAILQ_INSERT_TAIL(&process->finish_actions, finish_action, link);
    1685             : 
    1686           0 :         return 0;
    1687             : }
    1688             : 
    1689             : static void
    1690           0 : raid_bdev_unregistering_stop_process(void *ctx)
    1691             : {
    1692           0 :         struct raid_bdev_process *process = ctx;
    1693           0 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    1694             :         int rc;
    1695             : 
    1696           0 :         process->state = RAID_PROCESS_STATE_STOPPING;
    1697           0 :         if (process->status == 0) {
    1698           0 :                 process->status = -ECANCELED;
    1699             :         }
    1700             : 
    1701           0 :         rc = raid_bdev_process_add_finish_action(process, raid_bdev_unregistering_cont, raid_bdev);
    1702           0 :         if (rc != 0) {
    1703           0 :                 SPDK_ERRLOG("Failed to add raid bdev '%s' process finish action: %s\n",
    1704             :                             raid_bdev->bdev.name, spdk_strerror(-rc));
    1705             :         }
    1706           0 : }
    1707             : 
    1708             : static void
    1709           0 : raid_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx)
    1710             : {
    1711           0 :         struct raid_bdev *raid_bdev = event_ctx;
    1712             : 
    1713           0 :         if (type == SPDK_BDEV_EVENT_REMOVE) {
    1714           0 :                 if (raid_bdev->process != NULL) {
    1715           0 :                         spdk_thread_send_msg(raid_bdev->process->thread, raid_bdev_unregistering_stop_process,
    1716           0 :                                              raid_bdev->process);
    1717             :                 } else {
    1718           0 :                         raid_bdev_unregistering_cont(raid_bdev);
    1719             :                 }
    1720             :         }
    1721           0 : }
    1722             : 
    1723             : static void
    1724          14 : raid_bdev_configure_cont(struct raid_bdev *raid_bdev)
    1725             : {
    1726          14 :         struct spdk_bdev *raid_bdev_gen = &raid_bdev->bdev;
    1727             :         int rc;
    1728             : 
    1729          14 :         raid_bdev->state = RAID_BDEV_STATE_ONLINE;
    1730          14 :         SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
    1731          14 :         SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
    1732             :                       raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
    1733          14 :         spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
    1734             :                                 sizeof(struct raid_bdev_io_channel),
    1735          14 :                                 raid_bdev_gen->name);
    1736          14 :         rc = spdk_bdev_register(raid_bdev_gen);
    1737          14 :         if (rc != 0) {
    1738           0 :                 SPDK_ERRLOG("Failed to register raid bdev '%s': %s\n",
    1739             :                             raid_bdev_gen->name, spdk_strerror(-rc));
    1740           0 :                 goto out;
    1741             :         }
    1742             : 
    1743             :         /*
    1744             :          * Open the bdev internally to delay unregistering if we need to stop a background process
    1745             :          * first. The process may still need to unquiesce a range but it will fail because the
    1746             :          * bdev's internal.spinlock is destroyed by the time the destruct callback is reached.
    1747             :          * During application shutdown, bdevs automatically get unregistered by the bdev layer
    1748             :          * so this is the only way currently to do this correctly.
    1749             :          * TODO: try to handle this correctly in bdev layer instead.
    1750             :          */
    1751          14 :         rc = spdk_bdev_open_ext(raid_bdev_gen->name, false, raid_bdev_event_cb, raid_bdev,
    1752             :                                 &raid_bdev->self_desc);
    1753          14 :         if (rc != 0) {
    1754           0 :                 SPDK_ERRLOG("Failed to open raid bdev '%s': %s\n",
    1755             :                             raid_bdev_gen->name, spdk_strerror(-rc));
    1756           0 :                 spdk_bdev_unregister(raid_bdev_gen, NULL, NULL);
    1757           0 :                 goto out;
    1758             :         }
    1759             : 
    1760          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
    1761          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
    1762             :                       raid_bdev_gen->name, raid_bdev);
    1763          14 : out:
    1764          14 :         if (rc != 0) {
    1765           0 :                 if (raid_bdev->module->stop != NULL) {
    1766           0 :                         raid_bdev->module->stop(raid_bdev);
    1767             :                 }
    1768           0 :                 spdk_io_device_unregister(raid_bdev, NULL);
    1769           0 :                 raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
    1770             :         }
    1771             : 
    1772          14 :         if (raid_bdev->configure_cb != NULL) {
    1773          14 :                 raid_bdev->configure_cb(raid_bdev->configure_cb_ctx, rc);
    1774          14 :                 raid_bdev->configure_cb = NULL;
    1775             :         }
    1776          14 : }
    1777             : 
    1778             : static void
    1779           1 : raid_bdev_configure_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    1780             : {
    1781           1 :         if (status == 0) {
    1782           1 :                 raid_bdev_configure_cont(raid_bdev);
    1783             :         } else {
    1784           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock: %s\n",
    1785             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    1786           0 :                 if (raid_bdev->module->stop != NULL) {
    1787           0 :                         raid_bdev->module->stop(raid_bdev);
    1788             :                 }
    1789           0 :                 if (raid_bdev->configure_cb != NULL) {
    1790           0 :                         raid_bdev->configure_cb(raid_bdev->configure_cb_ctx, status);
    1791           0 :                         raid_bdev->configure_cb = NULL;
    1792             :                 }
    1793             :         }
    1794           1 : }
    1795             : 
    1796             : /*
    1797             :  * brief:
    1798             :  * If raid bdev config is complete, then only register the raid bdev to
    1799             :  * bdev layer and remove this raid bdev from configuring list and
    1800             :  * insert the raid bdev to configured list
    1801             :  * params:
    1802             :  * raid_bdev - pointer to raid bdev
    1803             :  * returns:
    1804             :  * 0 - success
    1805             :  * non zero - failure
    1806             :  */
    1807             : static int
    1808          14 : raid_bdev_configure(struct raid_bdev *raid_bdev, raid_bdev_configure_cb cb, void *cb_ctx)
    1809             : {
    1810          14 :         uint32_t data_block_size = spdk_bdev_get_data_block_size(&raid_bdev->bdev);
    1811             :         int rc;
    1812             : 
    1813          14 :         assert(raid_bdev->state == RAID_BDEV_STATE_CONFIGURING);
    1814          14 :         assert(raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational);
    1815          14 :         assert(raid_bdev->bdev.blocklen > 0);
    1816             : 
    1817             :         /* The strip_size_kb is read in from user in KB. Convert to blocks here for
    1818             :          * internal use.
    1819             :          */
    1820          14 :         raid_bdev->strip_size = (raid_bdev->strip_size_kb * 1024) / data_block_size;
    1821          14 :         if (raid_bdev->strip_size == 0 && raid_bdev->level != RAID1) {
    1822           0 :                 SPDK_ERRLOG("Strip size cannot be smaller than the device block size\n");
    1823           0 :                 return -EINVAL;
    1824             :         }
    1825          14 :         raid_bdev->strip_size_shift = spdk_u32log2(raid_bdev->strip_size);
    1826             : 
    1827          14 :         rc = raid_bdev->module->start(raid_bdev);
    1828          14 :         if (rc != 0) {
    1829           0 :                 SPDK_ERRLOG("raid module startup callback failed\n");
    1830           0 :                 return rc;
    1831             :         }
    1832             : 
    1833          14 :         assert(raid_bdev->configure_cb == NULL);
    1834          14 :         raid_bdev->configure_cb = cb;
    1835          14 :         raid_bdev->configure_cb_ctx = cb_ctx;
    1836             : 
    1837          14 :         if (raid_bdev->superblock_enabled) {
    1838           1 :                 if (raid_bdev->sb == NULL) {
    1839           1 :                         rc = raid_bdev_alloc_superblock(raid_bdev, data_block_size);
    1840           1 :                         if (rc == 0) {
    1841           1 :                                 raid_bdev_init_superblock(raid_bdev);
    1842             :                         }
    1843             :                 } else {
    1844           0 :                         assert(spdk_uuid_compare(&raid_bdev->sb->uuid, &raid_bdev->bdev.uuid) == 0);
    1845           0 :                         if (raid_bdev->sb->block_size != data_block_size) {
    1846           0 :                                 SPDK_ERRLOG("blocklen does not match value in superblock\n");
    1847           0 :                                 rc = -EINVAL;
    1848             :                         }
    1849           0 :                         if (raid_bdev->sb->raid_size != raid_bdev->bdev.blockcnt) {
    1850           0 :                                 SPDK_ERRLOG("blockcnt does not match value in superblock\n");
    1851           0 :                                 rc = -EINVAL;
    1852             :                         }
    1853             :                 }
    1854             : 
    1855           1 :                 if (rc != 0) {
    1856           0 :                         raid_bdev->configure_cb = NULL;
    1857           0 :                         if (raid_bdev->module->stop != NULL) {
    1858           0 :                                 raid_bdev->module->stop(raid_bdev);
    1859             :                         }
    1860           0 :                         return rc;
    1861             :                 }
    1862             : 
    1863           1 :                 raid_bdev_write_superblock(raid_bdev, raid_bdev_configure_write_sb_cb, NULL);
    1864             :         } else {
    1865          13 :                 raid_bdev_configure_cont(raid_bdev);
    1866             :         }
    1867             : 
    1868          14 :         return 0;
    1869             : }
    1870             : 
    1871             : /*
    1872             :  * brief:
    1873             :  * If raid bdev is online and registered, change the bdev state to
    1874             :  * configuring and unregister this raid device. Queue this raid device
    1875             :  * in configuring list
    1876             :  * params:
    1877             :  * raid_bdev - pointer to raid bdev
    1878             :  * cb_fn - callback function
    1879             :  * cb_arg - argument to callback function
    1880             :  * returns:
    1881             :  * none
    1882             :  */
    1883             : static void
    1884          14 : raid_bdev_deconfigure(struct raid_bdev *raid_bdev, raid_bdev_destruct_cb cb_fn,
    1885             :                       void *cb_arg)
    1886             : {
    1887          14 :         if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    1888           0 :                 if (cb_fn) {
    1889           0 :                         cb_fn(cb_arg, 0);
    1890             :                 }
    1891           0 :                 return;
    1892             :         }
    1893             : 
    1894          14 :         raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
    1895          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev state changing from online to offline\n");
    1896             : 
    1897          14 :         spdk_bdev_unregister(&raid_bdev->bdev, cb_fn, cb_arg);
    1898             : }
    1899             : 
    1900             : /*
    1901             :  * brief:
    1902             :  * raid_bdev_find_base_info_by_bdev function finds the base bdev info by bdev.
    1903             :  * params:
    1904             :  * base_bdev - pointer to base bdev
    1905             :  * returns:
    1906             :  * base bdev info if found, otherwise NULL.
    1907             :  */
    1908             : static struct raid_base_bdev_info *
    1909           0 : raid_bdev_find_base_info_by_bdev(struct spdk_bdev *base_bdev)
    1910             : {
    1911             :         struct raid_bdev *raid_bdev;
    1912             :         struct raid_base_bdev_info *base_info;
    1913             : 
    1914           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1915           0 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1916           0 :                         if (base_info->desc != NULL &&
    1917           0 :                             spdk_bdev_desc_get_bdev(base_info->desc) == base_bdev) {
    1918           0 :                                 return base_info;
    1919             :                         }
    1920             :                 }
    1921             :         }
    1922             : 
    1923           0 :         return NULL;
    1924             : }
    1925             : 
    1926             : static void
    1927           0 : raid_bdev_remove_base_bdev_done(struct raid_base_bdev_info *base_info, int status)
    1928             : {
    1929           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1930             : 
    1931           0 :         assert(base_info->remove_scheduled);
    1932           0 :         base_info->remove_scheduled = false;
    1933             : 
    1934           0 :         if (status == 0) {
    1935           0 :                 raid_bdev->num_base_bdevs_operational--;
    1936           0 :                 if (raid_bdev->num_base_bdevs_operational < raid_bdev->min_base_bdevs_operational) {
    1937             :                         /* There is not enough base bdevs to keep the raid bdev operational. */
    1938           0 :                         raid_bdev_deconfigure(raid_bdev, base_info->remove_cb, base_info->remove_cb_ctx);
    1939           0 :                         return;
    1940             :                 }
    1941             :         }
    1942             : 
    1943           0 :         if (base_info->remove_cb != NULL) {
    1944           0 :                 base_info->remove_cb(base_info->remove_cb_ctx, status);
    1945             :         }
    1946             : }
    1947             : 
    1948             : static void
    1949           0 : raid_bdev_remove_base_bdev_on_unquiesced(void *ctx, int status)
    1950             : {
    1951           0 :         struct raid_base_bdev_info *base_info = ctx;
    1952           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1953             : 
    1954           0 :         if (status != 0) {
    1955           0 :                 SPDK_ERRLOG("Failed to unquiesce raid bdev %s: %s\n",
    1956             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    1957             :         }
    1958             : 
    1959           0 :         raid_bdev_remove_base_bdev_done(base_info, status);
    1960           0 : }
    1961             : 
    1962             : static void
    1963           0 : raid_bdev_channel_remove_base_bdev(struct spdk_io_channel_iter *i)
    1964             : {
    1965           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    1966           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    1967           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    1968           0 :         uint8_t idx = raid_bdev_base_bdev_slot(base_info);
    1969             : 
    1970           0 :         SPDK_DEBUGLOG(bdev_raid, "slot: %u raid_ch: %p\n", idx, raid_ch);
    1971             : 
    1972           0 :         if (raid_ch->base_channel[idx] != NULL) {
    1973           0 :                 spdk_put_io_channel(raid_ch->base_channel[idx]);
    1974           0 :                 raid_ch->base_channel[idx] = NULL;
    1975             :         }
    1976             : 
    1977           0 :         if (raid_ch->process.ch_processed != NULL) {
    1978           0 :                 raid_ch->process.ch_processed->base_channel[idx] = NULL;
    1979             :         }
    1980             : 
    1981           0 :         spdk_for_each_channel_continue(i, 0);
    1982           0 : }
    1983             : 
    1984             : static void
    1985           0 : raid_bdev_channels_remove_base_bdev_done(struct spdk_io_channel_iter *i, int status)
    1986             : {
    1987           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    1988           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1989             : 
    1990           0 :         raid_bdev_free_base_bdev_resource(base_info);
    1991             : 
    1992           0 :         spdk_bdev_unquiesce(&raid_bdev->bdev, &g_raid_if, raid_bdev_remove_base_bdev_on_unquiesced,
    1993             :                             base_info);
    1994           0 : }
    1995             : 
    1996             : static void
    1997           0 : raid_bdev_remove_base_bdev_cont(struct raid_base_bdev_info *base_info)
    1998             : {
    1999           0 :         raid_bdev_deconfigure_base_bdev(base_info);
    2000             : 
    2001           0 :         spdk_for_each_channel(base_info->raid_bdev, raid_bdev_channel_remove_base_bdev, base_info,
    2002             :                               raid_bdev_channels_remove_base_bdev_done);
    2003           0 : }
    2004             : 
    2005             : static void
    2006           0 : raid_bdev_remove_base_bdev_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    2007             : {
    2008           0 :         struct raid_base_bdev_info *base_info = ctx;
    2009             : 
    2010           0 :         if (status != 0) {
    2011           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock: %s\n",
    2012             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2013           0 :                 raid_bdev_remove_base_bdev_done(base_info, status);
    2014           0 :                 return;
    2015             :         }
    2016             : 
    2017           0 :         raid_bdev_remove_base_bdev_cont(base_info);
    2018             : }
    2019             : 
    2020             : static void
    2021           0 : raid_bdev_remove_base_bdev_on_quiesced(void *ctx, int status)
    2022             : {
    2023           0 :         struct raid_base_bdev_info *base_info = ctx;
    2024           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    2025             : 
    2026           0 :         if (status != 0) {
    2027           0 :                 SPDK_ERRLOG("Failed to quiesce raid bdev %s: %s\n",
    2028             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2029           0 :                 raid_bdev_remove_base_bdev_done(base_info, status);
    2030           0 :                 return;
    2031             :         }
    2032             : 
    2033           0 :         if (raid_bdev->sb) {
    2034           0 :                 struct raid_bdev_superblock *sb = raid_bdev->sb;
    2035           0 :                 uint8_t slot = raid_bdev_base_bdev_slot(base_info);
    2036             :                 uint8_t i;
    2037             : 
    2038           0 :                 for (i = 0; i < sb->base_bdevs_size; i++) {
    2039           0 :                         struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    2040             : 
    2041           0 :                         if (sb_base_bdev->state == RAID_SB_BASE_BDEV_CONFIGURED &&
    2042           0 :                             sb_base_bdev->slot == slot) {
    2043           0 :                                 if (base_info->is_failed) {
    2044           0 :                                         sb_base_bdev->state = RAID_SB_BASE_BDEV_FAILED;
    2045             :                                 } else {
    2046           0 :                                         sb_base_bdev->state = RAID_SB_BASE_BDEV_MISSING;
    2047             :                                 }
    2048             : 
    2049           0 :                                 raid_bdev_write_superblock(raid_bdev, raid_bdev_remove_base_bdev_write_sb_cb, base_info);
    2050           0 :                                 return;
    2051             :                         }
    2052             :                 }
    2053             :         }
    2054             : 
    2055           0 :         raid_bdev_remove_base_bdev_cont(base_info);
    2056             : }
    2057             : 
    2058             : static int
    2059           0 : raid_bdev_remove_base_bdev_quiesce(struct raid_base_bdev_info *base_info)
    2060             : {
    2061           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2062             : 
    2063           0 :         return spdk_bdev_quiesce(&base_info->raid_bdev->bdev, &g_raid_if,
    2064             :                                  raid_bdev_remove_base_bdev_on_quiesced, base_info);
    2065             : }
    2066             : 
    2067             : struct raid_bdev_process_base_bdev_remove_ctx {
    2068             :         struct raid_bdev_process *process;
    2069             :         struct raid_base_bdev_info *base_info;
    2070             :         uint8_t num_base_bdevs_operational;
    2071             : };
    2072             : 
    2073             : static void
    2074           0 : _raid_bdev_process_base_bdev_remove_cont(void *ctx)
    2075             : {
    2076           0 :         struct raid_base_bdev_info *base_info = ctx;
    2077             :         int ret;
    2078             : 
    2079           0 :         ret = raid_bdev_remove_base_bdev_quiesce(base_info);
    2080           0 :         if (ret != 0) {
    2081           0 :                 raid_bdev_remove_base_bdev_done(base_info, ret);
    2082             :         }
    2083           0 : }
    2084             : 
    2085             : static void
    2086           0 : raid_bdev_process_base_bdev_remove_cont(void *_ctx)
    2087             : {
    2088           0 :         struct raid_bdev_process_base_bdev_remove_ctx *ctx = _ctx;
    2089           0 :         struct raid_base_bdev_info *base_info = ctx->base_info;
    2090             : 
    2091           0 :         free(ctx);
    2092             : 
    2093           0 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), _raid_bdev_process_base_bdev_remove_cont,
    2094             :                              base_info);
    2095           0 : }
    2096             : 
    2097             : static void
    2098           0 : _raid_bdev_process_base_bdev_remove(void *_ctx)
    2099             : {
    2100           0 :         struct raid_bdev_process_base_bdev_remove_ctx *ctx = _ctx;
    2101           0 :         struct raid_bdev_process *process = ctx->process;
    2102             :         int ret;
    2103             : 
    2104           0 :         if (ctx->base_info != process->target &&
    2105           0 :             ctx->num_base_bdevs_operational > process->raid_bdev->min_base_bdevs_operational) {
    2106             :                 /* process doesn't need to be stopped */
    2107           0 :                 raid_bdev_process_base_bdev_remove_cont(ctx);
    2108           0 :                 return;
    2109             :         }
    2110             : 
    2111           0 :         assert(process->state > RAID_PROCESS_STATE_INIT &&
    2112             :                process->state < RAID_PROCESS_STATE_STOPPED);
    2113             : 
    2114           0 :         ret = raid_bdev_process_add_finish_action(process, raid_bdev_process_base_bdev_remove_cont, ctx);
    2115           0 :         if (ret != 0) {
    2116           0 :                 raid_bdev_remove_base_bdev_done(ctx->base_info, ret);
    2117           0 :                 free(ctx);
    2118           0 :                 return;
    2119             :         }
    2120             : 
    2121           0 :         process->state = RAID_PROCESS_STATE_STOPPING;
    2122             : 
    2123           0 :         if (process->status == 0) {
    2124           0 :                 process->status = -ENODEV;
    2125             :         }
    2126             : }
    2127             : 
    2128             : static int
    2129           0 : raid_bdev_process_base_bdev_remove(struct raid_bdev_process *process,
    2130             :                                    struct raid_base_bdev_info *base_info)
    2131             : {
    2132             :         struct raid_bdev_process_base_bdev_remove_ctx *ctx;
    2133             : 
    2134           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2135             : 
    2136           0 :         ctx = calloc(1, sizeof(*ctx));
    2137           0 :         if (ctx == NULL) {
    2138           0 :                 return -ENOMEM;
    2139             :         }
    2140             : 
    2141             :         /*
    2142             :          * We have to send the process and num_base_bdevs_operational in the message ctx
    2143             :          * because the process thread should not access raid_bdev's properties. Particularly,
    2144             :          * raid_bdev->process may be cleared by the time the message is handled, but ctx->process
    2145             :          * will still be valid until the process is fully stopped.
    2146             :          */
    2147           0 :         ctx->base_info = base_info;
    2148           0 :         ctx->process = process;
    2149             :         /*
    2150             :          * raid_bdev->num_base_bdevs_operational can't be used here because it is decremented
    2151             :          * after the removal and more than one base bdev may be removed at the same time
    2152             :          */
    2153           0 :         RAID_FOR_EACH_BASE_BDEV(process->raid_bdev, base_info) {
    2154           0 :                 if (base_info->is_configured && !base_info->remove_scheduled) {
    2155           0 :                         ctx->num_base_bdevs_operational++;
    2156             :                 }
    2157             :         }
    2158             : 
    2159           0 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_base_bdev_remove, ctx);
    2160             : 
    2161           0 :         return 0;
    2162             : }
    2163             : 
    2164             : static int
    2165           0 : _raid_bdev_remove_base_bdev(struct raid_base_bdev_info *base_info,
    2166             :                             raid_base_bdev_cb cb_fn, void *cb_ctx)
    2167             : {
    2168           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    2169           0 :         int ret = 0;
    2170             : 
    2171           0 :         SPDK_DEBUGLOG(bdev_raid, "%s\n", base_info->name);
    2172             : 
    2173           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2174             : 
    2175           0 :         if (base_info->remove_scheduled || !base_info->is_configured) {
    2176           0 :                 return -ENODEV;
    2177             :         }
    2178             : 
    2179           0 :         assert(base_info->desc);
    2180           0 :         base_info->remove_scheduled = true;
    2181             : 
    2182           0 :         if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    2183             :                 /*
    2184             :                  * As raid bdev is not registered yet or already unregistered,
    2185             :                  * so cleanup should be done here itself.
    2186             :                  *
    2187             :                  * Removing a base bdev at this stage does not change the number of operational
    2188             :                  * base bdevs, only the number of discovered base bdevs.
    2189             :                  */
    2190           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    2191           0 :                 base_info->remove_scheduled = false;
    2192           0 :                 if (raid_bdev->num_base_bdevs_discovered == 0 &&
    2193           0 :                     raid_bdev->state == RAID_BDEV_STATE_OFFLINE) {
    2194             :                         /* There is no base bdev for this raid, so free the raid device. */
    2195           0 :                         raid_bdev_cleanup_and_free(raid_bdev);
    2196             :                 }
    2197           0 :                 if (cb_fn != NULL) {
    2198           0 :                         cb_fn(cb_ctx, 0);
    2199             :                 }
    2200           0 :         } else if (raid_bdev->min_base_bdevs_operational == raid_bdev->num_base_bdevs) {
    2201             :                 /* This raid bdev does not tolerate removing a base bdev. */
    2202           0 :                 raid_bdev->num_base_bdevs_operational--;
    2203           0 :                 raid_bdev_deconfigure(raid_bdev, cb_fn, cb_ctx);
    2204             :         } else {
    2205           0 :                 base_info->remove_cb = cb_fn;
    2206           0 :                 base_info->remove_cb_ctx = cb_ctx;
    2207             : 
    2208           0 :                 if (raid_bdev->process != NULL) {
    2209           0 :                         ret = raid_bdev_process_base_bdev_remove(raid_bdev->process, base_info);
    2210             :                 } else {
    2211           0 :                         ret = raid_bdev_remove_base_bdev_quiesce(base_info);
    2212             :                 }
    2213             : 
    2214           0 :                 if (ret != 0) {
    2215           0 :                         base_info->remove_scheduled = false;
    2216             :                 }
    2217             :         }
    2218             : 
    2219           0 :         return ret;
    2220             : }
    2221             : 
    2222             : /*
    2223             :  * brief:
    2224             :  * raid_bdev_remove_base_bdev function is called by below layers when base_bdev
    2225             :  * is removed. This function checks if this base bdev is part of any raid bdev
    2226             :  * or not. If yes, it takes necessary action on that particular raid bdev.
    2227             :  * params:
    2228             :  * base_bdev - pointer to base bdev which got removed
    2229             :  * cb_fn - callback function
    2230             :  * cb_arg - argument to callback function
    2231             :  * returns:
    2232             :  * 0 - success
    2233             :  * non zero - failure
    2234             :  */
    2235             : int
    2236           0 : raid_bdev_remove_base_bdev(struct spdk_bdev *base_bdev, raid_base_bdev_cb cb_fn, void *cb_ctx)
    2237             : {
    2238             :         struct raid_base_bdev_info *base_info;
    2239             : 
    2240             :         /* Find the raid_bdev which has claimed this base_bdev */
    2241           0 :         base_info = raid_bdev_find_base_info_by_bdev(base_bdev);
    2242           0 :         if (!base_info) {
    2243           0 :                 SPDK_ERRLOG("bdev to remove '%s' not found\n", base_bdev->name);
    2244           0 :                 return -ENODEV;
    2245             :         }
    2246             : 
    2247           0 :         return _raid_bdev_remove_base_bdev(base_info, cb_fn, cb_ctx);
    2248             : }
    2249             : 
    2250             : static void
    2251           0 : raid_bdev_fail_base_remove_cb(void *ctx, int status)
    2252             : {
    2253           0 :         struct raid_base_bdev_info *base_info = ctx;
    2254             : 
    2255           0 :         if (status != 0) {
    2256           0 :                 SPDK_WARNLOG("Failed to remove base bdev %s\n", base_info->name);
    2257           0 :                 base_info->is_failed = false;
    2258             :         }
    2259           0 : }
    2260             : 
    2261             : static void
    2262           0 : _raid_bdev_fail_base_bdev(void *ctx)
    2263             : {
    2264           0 :         struct raid_base_bdev_info *base_info = ctx;
    2265             :         int rc;
    2266             : 
    2267           0 :         if (base_info->is_failed) {
    2268           0 :                 return;
    2269             :         }
    2270           0 :         base_info->is_failed = true;
    2271             : 
    2272           0 :         SPDK_NOTICELOG("Failing base bdev in slot %d ('%s') of raid bdev '%s'\n",
    2273             :                        raid_bdev_base_bdev_slot(base_info), base_info->name, base_info->raid_bdev->bdev.name);
    2274             : 
    2275           0 :         rc = _raid_bdev_remove_base_bdev(base_info, raid_bdev_fail_base_remove_cb, base_info);
    2276           0 :         if (rc != 0) {
    2277           0 :                 raid_bdev_fail_base_remove_cb(base_info, rc);
    2278             :         }
    2279             : }
    2280             : 
    2281             : void
    2282           0 : raid_bdev_fail_base_bdev(struct raid_base_bdev_info *base_info)
    2283             : {
    2284           0 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_fail_base_bdev, base_info);
    2285           0 : }
    2286             : 
    2287             : static void
    2288           0 : raid_bdev_resize_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    2289             : {
    2290           0 :         if (status != 0) {
    2291           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock after resizing the bdev: %s\n",
    2292             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2293             :         }
    2294           0 : }
    2295             : 
    2296             : /*
    2297             :  * brief:
    2298             :  * raid_bdev_resize_base_bdev function is called by below layers when base_bdev
    2299             :  * is resized. This function checks if the smallest size of the base_bdevs is changed.
    2300             :  * If yes, call module handler to resize the raid_bdev if implemented.
    2301             :  * params:
    2302             :  * base_bdev - pointer to base bdev which got resized.
    2303             :  * returns:
    2304             :  * none
    2305             :  */
    2306             : static void
    2307           0 : raid_bdev_resize_base_bdev(struct spdk_bdev *base_bdev)
    2308             : {
    2309             :         struct raid_bdev *raid_bdev;
    2310             :         struct raid_base_bdev_info *base_info;
    2311             :         uint64_t blockcnt_old;
    2312             : 
    2313           0 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_resize_base_bdev\n");
    2314             : 
    2315           0 :         base_info = raid_bdev_find_base_info_by_bdev(base_bdev);
    2316             : 
    2317             :         /* Find the raid_bdev which has claimed this base_bdev */
    2318           0 :         if (!base_info) {
    2319           0 :                 SPDK_ERRLOG("raid_bdev whose base_bdev '%s' not found\n", base_bdev->name);
    2320           0 :                 return;
    2321             :         }
    2322           0 :         raid_bdev = base_info->raid_bdev;
    2323             : 
    2324           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2325             : 
    2326           0 :         SPDK_NOTICELOG("base_bdev '%s' was resized: old size %" PRIu64 ", new size %" PRIu64 "\n",
    2327             :                        base_bdev->name, base_info->blockcnt, base_bdev->blockcnt);
    2328             : 
    2329           0 :         base_info->blockcnt = base_bdev->blockcnt;
    2330             : 
    2331           0 :         if (!raid_bdev->module->resize) {
    2332           0 :                 return;
    2333             :         }
    2334             : 
    2335           0 :         blockcnt_old = raid_bdev->bdev.blockcnt;
    2336           0 :         if (raid_bdev->module->resize(raid_bdev) == false) {
    2337           0 :                 return;
    2338             :         }
    2339             : 
    2340           0 :         SPDK_NOTICELOG("raid bdev '%s': block count was changed from %" PRIu64 " to %" PRIu64 "\n",
    2341             :                        raid_bdev->bdev.name, blockcnt_old, raid_bdev->bdev.blockcnt);
    2342             : 
    2343           0 :         if (raid_bdev->superblock_enabled) {
    2344           0 :                 struct raid_bdev_superblock *sb = raid_bdev->sb;
    2345             :                 uint8_t i;
    2346             : 
    2347           0 :                 for (i = 0; i < sb->base_bdevs_size; i++) {
    2348           0 :                         struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    2349             : 
    2350           0 :                         if (sb_base_bdev->slot < raid_bdev->num_base_bdevs) {
    2351           0 :                                 base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    2352           0 :                                 sb_base_bdev->data_size = base_info->data_size;
    2353             :                         }
    2354             :                 }
    2355           0 :                 sb->raid_size = raid_bdev->bdev.blockcnt;
    2356           0 :                 raid_bdev_write_superblock(raid_bdev, raid_bdev_resize_write_sb_cb, NULL);
    2357             :         }
    2358             : }
    2359             : 
    2360             : /*
    2361             :  * brief:
    2362             :  * raid_bdev_event_base_bdev function is called by below layers when base_bdev
    2363             :  * triggers asynchronous event.
    2364             :  * params:
    2365             :  * type - event details.
    2366             :  * bdev - bdev that triggered event.
    2367             :  * event_ctx - context for event.
    2368             :  * returns:
    2369             :  * none
    2370             :  */
    2371             : static void
    2372           0 : raid_bdev_event_base_bdev(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
    2373             :                           void *event_ctx)
    2374             : {
    2375             :         int rc;
    2376             : 
    2377           0 :         switch (type) {
    2378           0 :         case SPDK_BDEV_EVENT_REMOVE:
    2379           0 :                 rc = raid_bdev_remove_base_bdev(bdev, NULL, NULL);
    2380           0 :                 if (rc != 0) {
    2381           0 :                         SPDK_ERRLOG("Failed to remove base bdev %s: %s\n",
    2382             :                                     spdk_bdev_get_name(bdev), spdk_strerror(-rc));
    2383             :                 }
    2384           0 :                 break;
    2385           0 :         case SPDK_BDEV_EVENT_RESIZE:
    2386           0 :                 raid_bdev_resize_base_bdev(bdev);
    2387           0 :                 break;
    2388           0 :         default:
    2389           0 :                 SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
    2390           0 :                 break;
    2391             :         }
    2392           0 : }
    2393             : 
    2394             : /*
    2395             :  * brief:
    2396             :  * Deletes the specified raid bdev
    2397             :  * params:
    2398             :  * raid_bdev - pointer to raid bdev
    2399             :  * cb_fn - callback function
    2400             :  * cb_arg - argument to callback function
    2401             :  */
    2402             : void
    2403          16 : raid_bdev_delete(struct raid_bdev *raid_bdev, raid_bdev_destruct_cb cb_fn, void *cb_arg)
    2404             : {
    2405             :         struct raid_base_bdev_info *base_info;
    2406             : 
    2407          16 :         SPDK_DEBUGLOG(bdev_raid, "delete raid bdev: %s\n", raid_bdev->bdev.name);
    2408             : 
    2409          16 :         if (raid_bdev->destroy_started) {
    2410           0 :                 SPDK_DEBUGLOG(bdev_raid, "destroying raid bdev %s is already started\n",
    2411             :                               raid_bdev->bdev.name);
    2412           0 :                 if (cb_fn) {
    2413           0 :                         cb_fn(cb_arg, -EALREADY);
    2414             :                 }
    2415           0 :                 return;
    2416             :         }
    2417             : 
    2418          16 :         raid_bdev->destroy_started = true;
    2419             : 
    2420         528 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    2421         512 :                 base_info->remove_scheduled = true;
    2422             : 
    2423         512 :                 if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    2424             :                         /*
    2425             :                          * As raid bdev is not registered yet or already unregistered,
    2426             :                          * so cleanup should be done here itself.
    2427             :                          */
    2428          64 :                         raid_bdev_free_base_bdev_resource(base_info);
    2429             :                 }
    2430             :         }
    2431             : 
    2432          16 :         if (raid_bdev->num_base_bdevs_discovered == 0) {
    2433             :                 /* There is no base bdev for this raid, so free the raid device. */
    2434           2 :                 raid_bdev_cleanup_and_free(raid_bdev);
    2435           2 :                 if (cb_fn) {
    2436           0 :                         cb_fn(cb_arg, 0);
    2437             :                 }
    2438             :         } else {
    2439          14 :                 raid_bdev_deconfigure(raid_bdev, cb_fn, cb_arg);
    2440             :         }
    2441             : }
    2442             : 
    2443             : static void
    2444           0 : raid_bdev_process_finish_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    2445             : {
    2446           0 :         if (status != 0) {
    2447           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock after background process finished: %s\n",
    2448             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2449             :         }
    2450           0 : }
    2451             : 
    2452             : static void
    2453           0 : raid_bdev_process_finish_write_sb(void *ctx)
    2454             : {
    2455           0 :         struct raid_bdev *raid_bdev = ctx;
    2456           0 :         struct raid_bdev_superblock *sb = raid_bdev->sb;
    2457             :         struct raid_bdev_sb_base_bdev *sb_base_bdev;
    2458             :         struct raid_base_bdev_info *base_info;
    2459             :         uint8_t i;
    2460             : 
    2461           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    2462           0 :                 sb_base_bdev = &sb->base_bdevs[i];
    2463             : 
    2464           0 :                 if (sb_base_bdev->state != RAID_SB_BASE_BDEV_CONFIGURED &&
    2465           0 :                     sb_base_bdev->slot < raid_bdev->num_base_bdevs) {
    2466           0 :                         base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    2467           0 :                         if (base_info->is_configured) {
    2468           0 :                                 sb_base_bdev->state = RAID_SB_BASE_BDEV_CONFIGURED;
    2469           0 :                                 sb_base_bdev->data_offset = base_info->data_offset;
    2470           0 :                                 spdk_uuid_copy(&sb_base_bdev->uuid, &base_info->uuid);
    2471             :                         }
    2472             :                 }
    2473             :         }
    2474             : 
    2475           0 :         raid_bdev_write_superblock(raid_bdev, raid_bdev_process_finish_write_sb_cb, NULL);
    2476           0 : }
    2477             : 
    2478             : static void raid_bdev_process_free(struct raid_bdev_process *process);
    2479             : 
    2480             : static void
    2481           2 : _raid_bdev_process_finish_done(void *ctx)
    2482             : {
    2483           2 :         struct raid_bdev_process *process = ctx;
    2484             :         struct raid_process_finish_action *finish_action;
    2485             : 
    2486           2 :         while ((finish_action = TAILQ_FIRST(&process->finish_actions)) != NULL) {
    2487           0 :                 TAILQ_REMOVE(&process->finish_actions, finish_action, link);
    2488           0 :                 finish_action->cb(finish_action->cb_ctx);
    2489           0 :                 free(finish_action);
    2490             :         }
    2491             : 
    2492           2 :         spdk_poller_unregister(&process->qos.process_continue_poller);
    2493             : 
    2494           2 :         raid_bdev_process_free(process);
    2495             : 
    2496           2 :         spdk_thread_exit(spdk_get_thread());
    2497           2 : }
    2498             : 
    2499             : static void
    2500           0 : raid_bdev_process_finish_target_removed(void *ctx, int status)
    2501             : {
    2502           0 :         struct raid_bdev_process *process = ctx;
    2503             : 
    2504           0 :         if (status != 0) {
    2505           0 :                 SPDK_ERRLOG("Failed to remove target bdev: %s\n", spdk_strerror(-status));
    2506             :         }
    2507             : 
    2508           0 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_finish_done, process);
    2509           0 : }
    2510             : 
    2511             : static void
    2512           2 : raid_bdev_process_finish_unquiesced(void *ctx, int status)
    2513             : {
    2514           2 :         struct raid_bdev_process *process = ctx;
    2515             : 
    2516           2 :         if (status != 0) {
    2517           0 :                 SPDK_ERRLOG("Failed to unquiesce bdev: %s\n", spdk_strerror(-status));
    2518             :         }
    2519             : 
    2520           2 :         if (process->status != 0) {
    2521           0 :                 status = _raid_bdev_remove_base_bdev(process->target, raid_bdev_process_finish_target_removed,
    2522             :                                                      process);
    2523           0 :                 if (status != 0) {
    2524           0 :                         raid_bdev_process_finish_target_removed(process, status);
    2525             :                 }
    2526           0 :                 return;
    2527             :         }
    2528             : 
    2529           2 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_finish_done, process);
    2530             : }
    2531             : 
    2532             : static void
    2533           2 : raid_bdev_process_finish_unquiesce(void *ctx)
    2534             : {
    2535           2 :         struct raid_bdev_process *process = ctx;
    2536             :         int rc;
    2537             : 
    2538           2 :         rc = spdk_bdev_unquiesce(&process->raid_bdev->bdev, &g_raid_if,
    2539             :                                  raid_bdev_process_finish_unquiesced, process);
    2540           2 :         if (rc != 0) {
    2541           0 :                 raid_bdev_process_finish_unquiesced(process, rc);
    2542             :         }
    2543           2 : }
    2544             : 
    2545             : static void
    2546           2 : raid_bdev_process_finish_done(void *ctx)
    2547             : {
    2548           2 :         struct raid_bdev_process *process = ctx;
    2549           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2550             : 
    2551           2 :         if (process->raid_ch != NULL) {
    2552           2 :                 spdk_put_io_channel(spdk_io_channel_from_ctx(process->raid_ch));
    2553             :         }
    2554             : 
    2555           2 :         process->state = RAID_PROCESS_STATE_STOPPED;
    2556             : 
    2557           2 :         if (process->status == 0) {
    2558           2 :                 SPDK_NOTICELOG("Finished %s on raid bdev %s\n",
    2559             :                                raid_bdev_process_to_str(process->type),
    2560             :                                raid_bdev->bdev.name);
    2561           2 :                 if (raid_bdev->superblock_enabled) {
    2562           0 :                         spdk_thread_send_msg(spdk_thread_get_app_thread(),
    2563             :                                              raid_bdev_process_finish_write_sb,
    2564             :                                              raid_bdev);
    2565             :                 }
    2566             :         } else {
    2567           0 :                 SPDK_WARNLOG("Finished %s on raid bdev %s: %s\n",
    2568             :                              raid_bdev_process_to_str(process->type),
    2569             :                              raid_bdev->bdev.name,
    2570             :                              spdk_strerror(-process->status));
    2571             :         }
    2572             : 
    2573           2 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), raid_bdev_process_finish_unquiesce,
    2574             :                              process);
    2575           2 : }
    2576             : 
    2577             : static void
    2578           2 : __raid_bdev_process_finish(struct spdk_io_channel_iter *i, int status)
    2579             : {
    2580           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2581             : 
    2582           2 :         spdk_thread_send_msg(process->thread, raid_bdev_process_finish_done, process);
    2583           2 : }
    2584             : 
    2585             : static void
    2586           2 : raid_bdev_channel_process_finish(struct spdk_io_channel_iter *i)
    2587             : {
    2588           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2589           2 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2590           2 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2591             : 
    2592           2 :         if (process->status == 0) {
    2593           2 :                 uint8_t slot = raid_bdev_base_bdev_slot(process->target);
    2594             : 
    2595           2 :                 raid_ch->base_channel[slot] = raid_ch->process.target_ch;
    2596           2 :                 raid_ch->process.target_ch = NULL;
    2597             :         }
    2598             : 
    2599           2 :         raid_bdev_ch_process_cleanup(raid_ch);
    2600             : 
    2601           2 :         spdk_for_each_channel_continue(i, 0);
    2602           2 : }
    2603             : 
    2604             : static void
    2605           2 : raid_bdev_process_finish_quiesced(void *ctx, int status)
    2606             : {
    2607           2 :         struct raid_bdev_process *process = ctx;
    2608           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2609             : 
    2610           2 :         if (status != 0) {
    2611           0 :                 SPDK_ERRLOG("Failed to quiesce bdev: %s\n", spdk_strerror(-status));
    2612           0 :                 return;
    2613             :         }
    2614             : 
    2615           2 :         raid_bdev->process = NULL;
    2616           2 :         process->target->is_process_target = false;
    2617             : 
    2618           2 :         spdk_for_each_channel(process->raid_bdev, raid_bdev_channel_process_finish, process,
    2619             :                               __raid_bdev_process_finish);
    2620             : }
    2621             : 
    2622             : static void
    2623           2 : _raid_bdev_process_finish(void *ctx)
    2624             : {
    2625           2 :         struct raid_bdev_process *process = ctx;
    2626             :         int rc;
    2627             : 
    2628           2 :         rc = spdk_bdev_quiesce(&process->raid_bdev->bdev, &g_raid_if,
    2629             :                                raid_bdev_process_finish_quiesced, process);
    2630           2 :         if (rc != 0) {
    2631           0 :                 raid_bdev_process_finish_quiesced(ctx, rc);
    2632             :         }
    2633           2 : }
    2634             : 
    2635             : static void
    2636           2 : raid_bdev_process_do_finish(struct raid_bdev_process *process)
    2637             : {
    2638           2 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), _raid_bdev_process_finish, process);
    2639           2 : }
    2640             : 
    2641             : static void raid_bdev_process_unlock_window_range(struct raid_bdev_process *process);
    2642             : static void raid_bdev_process_thread_run(struct raid_bdev_process *process);
    2643             : 
    2644             : static void
    2645           2 : raid_bdev_process_finish(struct raid_bdev_process *process, int status)
    2646             : {
    2647           2 :         assert(spdk_get_thread() == process->thread);
    2648             : 
    2649           2 :         if (process->status == 0) {
    2650           2 :                 process->status = status;
    2651             :         }
    2652             : 
    2653           2 :         if (process->state >= RAID_PROCESS_STATE_STOPPING) {
    2654           0 :                 return;
    2655             :         }
    2656             : 
    2657           2 :         assert(process->state == RAID_PROCESS_STATE_RUNNING);
    2658           2 :         process->state = RAID_PROCESS_STATE_STOPPING;
    2659             : 
    2660           2 :         if (process->window_range_locked) {
    2661           0 :                 raid_bdev_process_unlock_window_range(process);
    2662             :         } else {
    2663           2 :                 raid_bdev_process_thread_run(process);
    2664             :         }
    2665             : }
    2666             : 
    2667             : static void
    2668           2 : raid_bdev_process_window_range_unlocked(void *ctx, int status)
    2669             : {
    2670           2 :         struct raid_bdev_process *process = ctx;
    2671             : 
    2672           2 :         if (status != 0) {
    2673           0 :                 SPDK_ERRLOG("Failed to unlock LBA range: %s\n", spdk_strerror(-status));
    2674           0 :                 raid_bdev_process_finish(process, status);
    2675           0 :                 return;
    2676             :         }
    2677             : 
    2678           2 :         process->window_range_locked = false;
    2679           2 :         process->window_offset += process->window_size;
    2680             : 
    2681           2 :         raid_bdev_process_thread_run(process);
    2682             : }
    2683             : 
    2684             : static void
    2685           2 : raid_bdev_process_unlock_window_range(struct raid_bdev_process *process)
    2686             : {
    2687             :         int rc;
    2688             : 
    2689           2 :         assert(process->window_range_locked == true);
    2690             : 
    2691           2 :         rc = spdk_bdev_unquiesce_range(&process->raid_bdev->bdev, &g_raid_if,
    2692             :                                        process->window_offset, process->max_window_size,
    2693             :                                        raid_bdev_process_window_range_unlocked, process);
    2694           2 :         if (rc != 0) {
    2695           0 :                 raid_bdev_process_window_range_unlocked(process, rc);
    2696             :         }
    2697           2 : }
    2698             : 
    2699             : static void
    2700           2 : raid_bdev_process_channels_update_done(struct spdk_io_channel_iter *i, int status)
    2701             : {
    2702           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2703             : 
    2704           2 :         raid_bdev_process_unlock_window_range(process);
    2705           2 : }
    2706             : 
    2707             : static void
    2708           2 : raid_bdev_process_channel_update(struct spdk_io_channel_iter *i)
    2709             : {
    2710           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2711           2 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2712           2 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2713             : 
    2714           2 :         raid_ch->process.offset = process->window_offset + process->window_size;
    2715             : 
    2716           2 :         spdk_for_each_channel_continue(i, 0);
    2717           2 : }
    2718             : 
    2719             : void
    2720           2 : raid_bdev_process_request_complete(struct raid_bdev_process_request *process_req, int status)
    2721             : {
    2722           2 :         struct raid_bdev_process *process = process_req->process;
    2723             : 
    2724           2 :         TAILQ_INSERT_TAIL(&process->requests, process_req, link);
    2725             : 
    2726           2 :         assert(spdk_get_thread() == process->thread);
    2727           2 :         assert(process->window_remaining >= process_req->num_blocks);
    2728             : 
    2729           2 :         if (status != 0) {
    2730           0 :                 process->window_status = status;
    2731             :         }
    2732             : 
    2733           2 :         process->window_remaining -= process_req->num_blocks;
    2734           2 :         if (process->window_remaining == 0) {
    2735           2 :                 if (process->window_status != 0) {
    2736           0 :                         raid_bdev_process_finish(process, process->window_status);
    2737           0 :                         return;
    2738             :                 }
    2739             : 
    2740           2 :                 spdk_for_each_channel(process->raid_bdev, raid_bdev_process_channel_update, process,
    2741             :                                       raid_bdev_process_channels_update_done);
    2742             :         }
    2743             : }
    2744             : 
    2745             : static int
    2746           2 : raid_bdev_submit_process_request(struct raid_bdev_process *process, uint64_t offset_blocks,
    2747             :                                  uint32_t num_blocks)
    2748             : {
    2749           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2750             :         struct raid_bdev_process_request *process_req;
    2751             :         int ret;
    2752             : 
    2753           2 :         process_req = TAILQ_FIRST(&process->requests);
    2754           2 :         if (process_req == NULL) {
    2755           0 :                 assert(process->window_remaining > 0);
    2756           0 :                 return 0;
    2757             :         }
    2758             : 
    2759           2 :         process_req->target = process->target;
    2760           2 :         process_req->target_ch = process->raid_ch->process.target_ch;
    2761           2 :         process_req->offset_blocks = offset_blocks;
    2762           2 :         process_req->num_blocks = num_blocks;
    2763           2 :         process_req->iov.iov_len = num_blocks * raid_bdev->bdev.blocklen;
    2764             : 
    2765           2 :         ret = raid_bdev->module->submit_process_request(process_req, process->raid_ch);
    2766           2 :         if (ret <= 0) {
    2767           0 :                 if (ret < 0) {
    2768           0 :                         SPDK_ERRLOG("Failed to submit process request on %s: %s\n",
    2769             :                                     raid_bdev->bdev.name, spdk_strerror(-ret));
    2770           0 :                         process->window_status = ret;
    2771             :                 }
    2772           0 :                 return ret;
    2773             :         }
    2774             : 
    2775           2 :         process_req->num_blocks = ret;
    2776           2 :         TAILQ_REMOVE(&process->requests, process_req, link);
    2777             : 
    2778           2 :         return ret;
    2779             : }
    2780             : 
    2781             : static void
    2782           2 : _raid_bdev_process_thread_run(struct raid_bdev_process *process)
    2783             : {
    2784           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2785           2 :         uint64_t offset = process->window_offset;
    2786           2 :         const uint64_t offset_end = spdk_min(offset + process->max_window_size, raid_bdev->bdev.blockcnt);
    2787             :         int ret;
    2788             : 
    2789           4 :         while (offset < offset_end) {
    2790           2 :                 ret = raid_bdev_submit_process_request(process, offset, offset_end - offset);
    2791           2 :                 if (ret <= 0) {
    2792           0 :                         break;
    2793             :                 }
    2794             : 
    2795           2 :                 process->window_remaining += ret;
    2796           2 :                 offset += ret;
    2797             :         }
    2798             : 
    2799           2 :         if (process->window_remaining > 0) {
    2800           2 :                 process->window_size = process->window_remaining;
    2801             :         } else {
    2802           0 :                 raid_bdev_process_finish(process, process->window_status);
    2803             :         }
    2804           2 : }
    2805             : 
    2806             : static void
    2807           2 : raid_bdev_process_window_range_locked(void *ctx, int status)
    2808             : {
    2809           2 :         struct raid_bdev_process *process = ctx;
    2810             : 
    2811           2 :         if (status != 0) {
    2812           0 :                 SPDK_ERRLOG("Failed to lock LBA range: %s\n", spdk_strerror(-status));
    2813           0 :                 raid_bdev_process_finish(process, status);
    2814           0 :                 return;
    2815             :         }
    2816             : 
    2817           2 :         process->window_range_locked = true;
    2818             : 
    2819           2 :         if (process->state == RAID_PROCESS_STATE_STOPPING) {
    2820           0 :                 raid_bdev_process_unlock_window_range(process);
    2821           0 :                 return;
    2822             :         }
    2823             : 
    2824           2 :         _raid_bdev_process_thread_run(process);
    2825             : }
    2826             : 
    2827             : static bool
    2828          12 : raid_bdev_process_consume_token(struct raid_bdev_process *process)
    2829             : {
    2830          12 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2831          12 :         uint64_t now = spdk_get_ticks();
    2832             : 
    2833          12 :         process->qos.bytes_available = spdk_min(process->qos.bytes_max,
    2834             :                                                 process->qos.bytes_available +
    2835             :                                                 (now - process->qos.last_tsc) * process->qos.bytes_per_tsc);
    2836          12 :         process->qos.last_tsc = now;
    2837          12 :         if (process->qos.bytes_available > 0.0) {
    2838           1 :                 process->qos.bytes_available -= process->window_size * raid_bdev->bdev.blocklen;
    2839           1 :                 return true;
    2840             :         }
    2841          11 :         return false;
    2842             : }
    2843             : 
    2844             : static bool
    2845          13 : raid_bdev_process_lock_window_range(struct raid_bdev_process *process)
    2846             : {
    2847          13 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2848             :         int rc;
    2849             : 
    2850          13 :         assert(process->window_range_locked == false);
    2851             : 
    2852          13 :         if (process->qos.enable_qos) {
    2853          12 :                 if (raid_bdev_process_consume_token(process)) {
    2854           1 :                         spdk_poller_pause(process->qos.process_continue_poller);
    2855             :                 } else {
    2856          11 :                         spdk_poller_resume(process->qos.process_continue_poller);
    2857          11 :                         return false;
    2858             :                 }
    2859             :         }
    2860             : 
    2861           2 :         rc = spdk_bdev_quiesce_range(&raid_bdev->bdev, &g_raid_if,
    2862             :                                      process->window_offset, process->max_window_size,
    2863             :                                      raid_bdev_process_window_range_locked, process);
    2864           2 :         if (rc != 0) {
    2865           0 :                 raid_bdev_process_window_range_locked(process, rc);
    2866             :         }
    2867           2 :         return true;
    2868             : }
    2869             : 
    2870             : static int
    2871          11 : raid_bdev_process_continue_poll(void *arg)
    2872             : {
    2873          11 :         struct raid_bdev_process *process = arg;
    2874             : 
    2875          11 :         if (raid_bdev_process_lock_window_range(process)) {
    2876           1 :                 return SPDK_POLLER_BUSY;
    2877             :         }
    2878          10 :         return SPDK_POLLER_IDLE;
    2879             : }
    2880             : 
    2881             : static void
    2882           6 : raid_bdev_process_thread_run(struct raid_bdev_process *process)
    2883             : {
    2884           6 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2885             : 
    2886           6 :         assert(spdk_get_thread() == process->thread);
    2887           6 :         assert(process->window_remaining == 0);
    2888           6 :         assert(process->window_range_locked == false);
    2889             : 
    2890           6 :         if (process->state == RAID_PROCESS_STATE_STOPPING) {
    2891           2 :                 raid_bdev_process_do_finish(process);
    2892           2 :                 return;
    2893             :         }
    2894             : 
    2895           4 :         if (process->window_offset == raid_bdev->bdev.blockcnt) {
    2896           2 :                 SPDK_DEBUGLOG(bdev_raid, "process completed on %s\n", raid_bdev->bdev.name);
    2897           2 :                 raid_bdev_process_finish(process, 0);
    2898           2 :                 return;
    2899             :         }
    2900             : 
    2901           2 :         process->max_window_size = spdk_min(raid_bdev->bdev.blockcnt - process->window_offset,
    2902             :                                             process->max_window_size);
    2903           2 :         raid_bdev_process_lock_window_range(process);
    2904             : }
    2905             : 
    2906             : static void
    2907           2 : raid_bdev_process_thread_init(void *ctx)
    2908             : {
    2909           2 :         struct raid_bdev_process *process = ctx;
    2910           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2911             :         struct spdk_io_channel *ch;
    2912             : 
    2913           2 :         process->thread = spdk_get_thread();
    2914             : 
    2915           2 :         ch = spdk_get_io_channel(raid_bdev);
    2916           2 :         if (ch == NULL) {
    2917           0 :                 process->status = -ENOMEM;
    2918           0 :                 raid_bdev_process_do_finish(process);
    2919           0 :                 return;
    2920             :         }
    2921             : 
    2922           2 :         process->raid_ch = spdk_io_channel_get_ctx(ch);
    2923           2 :         process->state = RAID_PROCESS_STATE_RUNNING;
    2924             : 
    2925           2 :         if (process->qos.enable_qos) {
    2926           1 :                 process->qos.process_continue_poller = SPDK_POLLER_REGISTER(raid_bdev_process_continue_poll,
    2927             :                                                        process, 0);
    2928           1 :                 spdk_poller_pause(process->qos.process_continue_poller);
    2929             :         }
    2930             : 
    2931           2 :         SPDK_NOTICELOG("Started %s on raid bdev %s\n",
    2932             :                        raid_bdev_process_to_str(process->type), raid_bdev->bdev.name);
    2933             : 
    2934           2 :         raid_bdev_process_thread_run(process);
    2935             : }
    2936             : 
    2937             : static void
    2938           0 : raid_bdev_channels_abort_start_process_done(struct spdk_io_channel_iter *i, int status)
    2939             : {
    2940           0 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2941             : 
    2942           0 :         _raid_bdev_remove_base_bdev(process->target, NULL, NULL);
    2943           0 :         raid_bdev_process_free(process);
    2944             : 
    2945             :         /* TODO: update sb */
    2946           0 : }
    2947             : 
    2948             : static void
    2949           0 : raid_bdev_channel_abort_start_process(struct spdk_io_channel_iter *i)
    2950             : {
    2951           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2952           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2953             : 
    2954           0 :         raid_bdev_ch_process_cleanup(raid_ch);
    2955             : 
    2956           0 :         spdk_for_each_channel_continue(i, 0);
    2957           0 : }
    2958             : 
    2959             : static void
    2960           2 : raid_bdev_channels_start_process_done(struct spdk_io_channel_iter *i, int status)
    2961             : {
    2962           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2963           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2964             :         struct spdk_thread *thread;
    2965           2 :         char thread_name[RAID_BDEV_SB_NAME_SIZE + 16];
    2966             : 
    2967           2 :         if (status == 0 &&
    2968           2 :             (process->target->remove_scheduled || !process->target->is_configured ||
    2969           2 :              raid_bdev->num_base_bdevs_operational <= raid_bdev->min_base_bdevs_operational)) {
    2970             :                 /* a base bdev was removed before we got here */
    2971           0 :                 status = -ENODEV;
    2972             :         }
    2973             : 
    2974           2 :         if (status != 0) {
    2975           0 :                 SPDK_ERRLOG("Failed to start %s on %s: %s\n",
    2976             :                             raid_bdev_process_to_str(process->type), raid_bdev->bdev.name,
    2977             :                             spdk_strerror(-status));
    2978           0 :                 goto err;
    2979             :         }
    2980             : 
    2981           2 :         snprintf(thread_name, sizeof(thread_name), "%s_%s",
    2982             :                  raid_bdev->bdev.name, raid_bdev_process_to_str(process->type));
    2983             : 
    2984           2 :         thread = spdk_thread_create(thread_name, NULL);
    2985           2 :         if (thread == NULL) {
    2986           0 :                 SPDK_ERRLOG("Failed to create %s thread for %s\n",
    2987             :                             raid_bdev_process_to_str(process->type), raid_bdev->bdev.name);
    2988           0 :                 goto err;
    2989             :         }
    2990             : 
    2991           2 :         raid_bdev->process = process;
    2992             : 
    2993           2 :         spdk_thread_send_msg(thread, raid_bdev_process_thread_init, process);
    2994             : 
    2995           2 :         return;
    2996           0 : err:
    2997           0 :         spdk_for_each_channel(process->raid_bdev, raid_bdev_channel_abort_start_process, process,
    2998             :                               raid_bdev_channels_abort_start_process_done);
    2999             : }
    3000             : 
    3001             : static void
    3002           0 : raid_bdev_channel_start_process(struct spdk_io_channel_iter *i)
    3003             : {
    3004           0 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    3005           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    3006           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    3007             :         int rc;
    3008             : 
    3009           0 :         rc = raid_bdev_ch_process_setup(raid_ch, process);
    3010             : 
    3011           0 :         spdk_for_each_channel_continue(i, rc);
    3012           0 : }
    3013             : 
    3014             : static void
    3015           2 : raid_bdev_process_start(struct raid_bdev_process *process)
    3016             : {
    3017           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    3018             : 
    3019           2 :         assert(raid_bdev->module->submit_process_request != NULL);
    3020             : 
    3021           2 :         spdk_for_each_channel(raid_bdev, raid_bdev_channel_start_process, process,
    3022             :                               raid_bdev_channels_start_process_done);
    3023           2 : }
    3024             : 
    3025             : static void
    3026          32 : raid_bdev_process_request_free(struct raid_bdev_process_request *process_req)
    3027             : {
    3028          32 :         spdk_dma_free(process_req->iov.iov_base);
    3029          32 :         spdk_dma_free(process_req->md_buf);
    3030          32 :         free(process_req);
    3031          32 : }
    3032             : 
    3033             : static struct raid_bdev_process_request *
    3034          32 : raid_bdev_process_alloc_request(struct raid_bdev_process *process)
    3035             : {
    3036          32 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    3037             :         struct raid_bdev_process_request *process_req;
    3038             : 
    3039          32 :         process_req = calloc(1, sizeof(*process_req));
    3040          32 :         if (process_req == NULL) {
    3041           0 :                 return NULL;
    3042             :         }
    3043             : 
    3044          32 :         process_req->process = process;
    3045          32 :         process_req->iov.iov_len = process->max_window_size * raid_bdev->bdev.blocklen;
    3046          32 :         process_req->iov.iov_base = spdk_dma_malloc(process_req->iov.iov_len, 4096, 0);
    3047          32 :         if (process_req->iov.iov_base == NULL) {
    3048           0 :                 free(process_req);
    3049           0 :                 return NULL;
    3050             :         }
    3051          32 :         if (spdk_bdev_is_md_separate(&raid_bdev->bdev)) {
    3052          32 :                 process_req->md_buf = spdk_dma_malloc(process->max_window_size * raid_bdev->bdev.md_len, 4096, 0);
    3053          32 :                 if (process_req->md_buf == NULL) {
    3054           0 :                         raid_bdev_process_request_free(process_req);
    3055           0 :                         return NULL;
    3056             :                 }
    3057             :         }
    3058             : 
    3059          32 :         return process_req;
    3060             : }
    3061             : 
    3062             : static void
    3063           2 : raid_bdev_process_free(struct raid_bdev_process *process)
    3064             : {
    3065             :         struct raid_bdev_process_request *process_req;
    3066             : 
    3067          34 :         while ((process_req = TAILQ_FIRST(&process->requests)) != NULL) {
    3068          32 :                 TAILQ_REMOVE(&process->requests, process_req, link);
    3069          32 :                 raid_bdev_process_request_free(process_req);
    3070             :         }
    3071             : 
    3072           2 :         free(process);
    3073           2 : }
    3074             : 
    3075             : static struct raid_bdev_process *
    3076           2 : raid_bdev_process_alloc(struct raid_bdev *raid_bdev, enum raid_process_type type,
    3077             :                         struct raid_base_bdev_info *target)
    3078             : {
    3079             :         struct raid_bdev_process *process;
    3080             :         struct raid_bdev_process_request *process_req;
    3081             :         int i;
    3082             : 
    3083           2 :         process = calloc(1, sizeof(*process));
    3084           2 :         if (process == NULL) {
    3085           0 :                 return NULL;
    3086             :         }
    3087             : 
    3088           2 :         process->raid_bdev = raid_bdev;
    3089           2 :         process->type = type;
    3090           2 :         process->target = target;
    3091           2 :         process->max_window_size = spdk_max(spdk_divide_round_up(g_opts.process_window_size_kb * 1024UL,
    3092             :                                             spdk_bdev_get_data_block_size(&raid_bdev->bdev)),
    3093             :                                             raid_bdev->bdev.write_unit_size);
    3094           2 :         TAILQ_INIT(&process->requests);
    3095           2 :         TAILQ_INIT(&process->finish_actions);
    3096             : 
    3097           2 :         if (g_opts.process_max_bandwidth_mb_sec != 0) {
    3098           1 :                 process->qos.enable_qos = true;
    3099           1 :                 process->qos.last_tsc = spdk_get_ticks();
    3100           2 :                 process->qos.bytes_per_tsc = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 /
    3101           1 :                                              spdk_get_ticks_hz();
    3102           1 :                 process->qos.bytes_max = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 / SPDK_SEC_TO_MSEC;
    3103           1 :                 process->qos.bytes_available = 0.0;
    3104             :         }
    3105             : 
    3106          34 :         for (i = 0; i < RAID_BDEV_PROCESS_MAX_QD; i++) {
    3107          32 :                 process_req = raid_bdev_process_alloc_request(process);
    3108          32 :                 if (process_req == NULL) {
    3109           0 :                         raid_bdev_process_free(process);
    3110           0 :                         return NULL;
    3111             :                 }
    3112             : 
    3113          32 :                 TAILQ_INSERT_TAIL(&process->requests, process_req, link);
    3114             :         }
    3115             : 
    3116           2 :         return process;
    3117             : }
    3118             : 
    3119             : static int
    3120           2 : raid_bdev_start_rebuild(struct raid_base_bdev_info *target)
    3121             : {
    3122             :         struct raid_bdev_process *process;
    3123             : 
    3124           2 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3125             : 
    3126           2 :         process = raid_bdev_process_alloc(target->raid_bdev, RAID_PROCESS_REBUILD, target);
    3127           2 :         if (process == NULL) {
    3128           0 :                 return -ENOMEM;
    3129             :         }
    3130             : 
    3131           2 :         raid_bdev_process_start(process);
    3132             : 
    3133           2 :         return 0;
    3134             : }
    3135             : 
    3136             : static void raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info);
    3137             : 
    3138             : static void
    3139           0 : _raid_bdev_configure_base_bdev_cont(struct spdk_io_channel_iter *i, int status)
    3140             : {
    3141           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    3142             : 
    3143           0 :         raid_bdev_configure_base_bdev_cont(base_info);
    3144           0 : }
    3145             : 
    3146             : static void
    3147           0 : raid_bdev_ch_sync(struct spdk_io_channel_iter *i)
    3148             : {
    3149           0 :         spdk_for_each_channel_continue(i, 0);
    3150           0 : }
    3151             : 
    3152             : static void
    3153         510 : raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info)
    3154             : {
    3155         510 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    3156             :         raid_base_bdev_cb configure_cb;
    3157             :         int rc;
    3158             : 
    3159         510 :         if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational &&
    3160           0 :             base_info->is_process_target == false) {
    3161             :                 /* TODO: defer if rebuild in progress on another base bdev */
    3162           0 :                 assert(raid_bdev->process == NULL);
    3163           0 :                 assert(raid_bdev->state == RAID_BDEV_STATE_ONLINE);
    3164           0 :                 base_info->is_process_target = true;
    3165             :                 /* To assure is_process_target is set before is_configured when checked in raid_bdev_create_cb() */
    3166           0 :                 spdk_for_each_channel(raid_bdev, raid_bdev_ch_sync, base_info, _raid_bdev_configure_base_bdev_cont);
    3167           0 :                 return;
    3168             :         }
    3169             : 
    3170         510 :         base_info->is_configured = true;
    3171             : 
    3172         510 :         raid_bdev->num_base_bdevs_discovered++;
    3173         510 :         assert(raid_bdev->num_base_bdevs_discovered <= raid_bdev->num_base_bdevs);
    3174         510 :         assert(raid_bdev->num_base_bdevs_operational <= raid_bdev->num_base_bdevs);
    3175         510 :         assert(raid_bdev->num_base_bdevs_operational >= raid_bdev->min_base_bdevs_operational);
    3176             : 
    3177         510 :         configure_cb = base_info->configure_cb;
    3178         510 :         base_info->configure_cb = NULL;
    3179             :         /*
    3180             :          * Configure the raid bdev when the number of discovered base bdevs reaches the number
    3181             :          * of base bdevs we know to be operational members of the array. Usually this is equal
    3182             :          * to the total number of base bdevs (num_base_bdevs) but can be less - when the array is
    3183             :          * degraded.
    3184             :          */
    3185         510 :         if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational) {
    3186          14 :                 rc = raid_bdev_configure(raid_bdev, configure_cb, base_info->configure_cb_ctx);
    3187          14 :                 if (rc != 0) {
    3188           0 :                         SPDK_ERRLOG("Failed to configure raid bdev: %s\n", spdk_strerror(-rc));
    3189             :                 } else {
    3190          14 :                         configure_cb = NULL;
    3191             :                 }
    3192         496 :         } else if (base_info->is_process_target) {
    3193           0 :                 raid_bdev->num_base_bdevs_operational++;
    3194           0 :                 rc = raid_bdev_start_rebuild(base_info);
    3195           0 :                 if (rc != 0) {
    3196           0 :                         SPDK_ERRLOG("Failed to start rebuild: %s\n", spdk_strerror(-rc));
    3197           0 :                         _raid_bdev_remove_base_bdev(base_info, NULL, NULL);
    3198             :                 }
    3199             :         } else {
    3200         496 :                 rc = 0;
    3201             :         }
    3202             : 
    3203         510 :         if (configure_cb != NULL) {
    3204         496 :                 configure_cb(base_info->configure_cb_ctx, rc);
    3205             :         }
    3206             : }
    3207             : 
    3208             : static void raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3209             :                                  raid_base_bdev_cb cb_fn, void *cb_ctx);
    3210             : 
    3211             : static void
    3212         510 : raid_bdev_configure_base_bdev_check_sb_cb(const struct raid_bdev_superblock *sb, int status,
    3213             :                 void *ctx)
    3214             : {
    3215         510 :         struct raid_base_bdev_info *base_info = ctx;
    3216         510 :         raid_base_bdev_cb configure_cb = base_info->configure_cb;
    3217             : 
    3218         510 :         switch (status) {
    3219           0 :         case 0:
    3220             :                 /* valid superblock found */
    3221           0 :                 base_info->configure_cb = NULL;
    3222           0 :                 if (spdk_uuid_compare(&base_info->raid_bdev->bdev.uuid, &sb->uuid) == 0) {
    3223           0 :                         struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(base_info->desc);
    3224             : 
    3225           0 :                         raid_bdev_free_base_bdev_resource(base_info);
    3226           0 :                         raid_bdev_examine_sb(sb, bdev, configure_cb, base_info->configure_cb_ctx);
    3227           0 :                         return;
    3228             :                 }
    3229           0 :                 SPDK_ERRLOG("Superblock of a different raid bdev found on bdev %s\n", base_info->name);
    3230           0 :                 status = -EEXIST;
    3231           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    3232           0 :                 break;
    3233         510 :         case -EINVAL:
    3234             :                 /* no valid superblock */
    3235         510 :                 raid_bdev_configure_base_bdev_cont(base_info);
    3236         510 :                 return;
    3237           0 :         default:
    3238           0 :                 SPDK_ERRLOG("Failed to examine bdev %s: %s\n",
    3239             :                             base_info->name, spdk_strerror(-status));
    3240           0 :                 break;
    3241             :         }
    3242             : 
    3243           0 :         if (configure_cb != NULL) {
    3244           0 :                 base_info->configure_cb = NULL;
    3245           0 :                 configure_cb(base_info->configure_cb_ctx, status);
    3246             :         }
    3247             : }
    3248             : 
    3249             : static int
    3250         513 : raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info, bool existing,
    3251             :                               raid_base_bdev_cb cb_fn, void *cb_ctx)
    3252             : {
    3253         513 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    3254         513 :         struct spdk_bdev_desc *desc;
    3255             :         struct spdk_bdev *bdev;
    3256             :         const struct spdk_uuid *bdev_uuid;
    3257             :         int rc;
    3258             : 
    3259         513 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3260         513 :         assert(base_info->desc == NULL);
    3261             : 
    3262             :         /*
    3263             :          * Base bdev can be added by name or uuid. Here we assure both properties are set and valid
    3264             :          * before claiming the bdev.
    3265             :          */
    3266             : 
    3267         513 :         if (!spdk_uuid_is_null(&base_info->uuid)) {
    3268           0 :                 char uuid_str[SPDK_UUID_STRING_LEN];
    3269             :                 const char *bdev_name;
    3270             : 
    3271           0 :                 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &base_info->uuid);
    3272             : 
    3273             :                 /* UUID of a bdev is registered as its alias */
    3274           0 :                 bdev = spdk_bdev_get_by_name(uuid_str);
    3275           0 :                 if (bdev == NULL) {
    3276           0 :                         return -ENODEV;
    3277             :                 }
    3278             : 
    3279           0 :                 bdev_name = spdk_bdev_get_name(bdev);
    3280             : 
    3281           0 :                 if (base_info->name == NULL) {
    3282           0 :                         assert(existing == true);
    3283           0 :                         base_info->name = strdup(bdev_name);
    3284           0 :                         if (base_info->name == NULL) {
    3285           0 :                                 return -ENOMEM;
    3286             :                         }
    3287           0 :                 } else if (strcmp(base_info->name, bdev_name) != 0) {
    3288           0 :                         SPDK_ERRLOG("Name mismatch for base bdev '%s' - expected '%s'\n",
    3289             :                                     bdev_name, base_info->name);
    3290           0 :                         return -EINVAL;
    3291             :                 }
    3292             :         }
    3293             : 
    3294         513 :         assert(base_info->name != NULL);
    3295             : 
    3296         513 :         rc = spdk_bdev_open_ext(base_info->name, true, raid_bdev_event_base_bdev, NULL, &desc);
    3297         513 :         if (rc != 0) {
    3298           1 :                 if (rc != -ENODEV) {
    3299           0 :                         SPDK_ERRLOG("Unable to create desc on bdev '%s'\n", base_info->name);
    3300             :                 }
    3301           1 :                 return rc;
    3302             :         }
    3303             : 
    3304         512 :         bdev = spdk_bdev_desc_get_bdev(desc);
    3305         512 :         bdev_uuid = spdk_bdev_get_uuid(bdev);
    3306             : 
    3307         512 :         if (spdk_uuid_is_null(&base_info->uuid)) {
    3308         512 :                 spdk_uuid_copy(&base_info->uuid, bdev_uuid);
    3309           0 :         } else if (spdk_uuid_compare(&base_info->uuid, bdev_uuid) != 0) {
    3310           0 :                 SPDK_ERRLOG("UUID mismatch for base bdev '%s'\n", base_info->name);
    3311           0 :                 spdk_bdev_close(desc);
    3312           0 :                 return -EINVAL;
    3313             :         }
    3314             : 
    3315         512 :         rc = spdk_bdev_module_claim_bdev(bdev, NULL, &g_raid_if);
    3316         512 :         if (rc != 0) {
    3317           2 :                 SPDK_ERRLOG("Unable to claim this bdev as it is already claimed\n");
    3318           2 :                 spdk_bdev_close(desc);
    3319           2 :                 return rc;
    3320             :         }
    3321             : 
    3322         510 :         SPDK_DEBUGLOG(bdev_raid, "bdev %s is claimed\n", bdev->name);
    3323             : 
    3324         510 :         base_info->app_thread_ch = spdk_bdev_get_io_channel(desc);
    3325         510 :         if (base_info->app_thread_ch == NULL) {
    3326           0 :                 SPDK_ERRLOG("Failed to get io channel\n");
    3327           0 :                 spdk_bdev_module_release_bdev(bdev);
    3328           0 :                 spdk_bdev_close(desc);
    3329           0 :                 return -ENOMEM;
    3330             :         }
    3331             : 
    3332         510 :         base_info->desc = desc;
    3333         510 :         base_info->blockcnt = bdev->blockcnt;
    3334             : 
    3335         510 :         if (raid_bdev->superblock_enabled) {
    3336             :                 uint64_t data_offset;
    3337             : 
    3338          32 :                 if (base_info->data_offset == 0) {
    3339          32 :                         assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % spdk_bdev_get_data_block_size(bdev)) == 0);
    3340          32 :                         data_offset = RAID_BDEV_MIN_DATA_OFFSET_SIZE / spdk_bdev_get_data_block_size(bdev);
    3341             :                 } else {
    3342           0 :                         data_offset = base_info->data_offset;
    3343             :                 }
    3344             : 
    3345          32 :                 if (bdev->optimal_io_boundary != 0) {
    3346           0 :                         data_offset = spdk_divide_round_up(data_offset,
    3347           0 :                                                            bdev->optimal_io_boundary) * bdev->optimal_io_boundary;
    3348           0 :                         if (base_info->data_offset != 0 && base_info->data_offset != data_offset) {
    3349           0 :                                 SPDK_WARNLOG("Data offset %lu on bdev '%s' is different than optimal value %lu\n",
    3350             :                                              base_info->data_offset, base_info->name, data_offset);
    3351           0 :                                 data_offset = base_info->data_offset;
    3352             :                         }
    3353             :                 }
    3354             : 
    3355          32 :                 base_info->data_offset = data_offset;
    3356             :         }
    3357             : 
    3358         510 :         if (base_info->data_offset >= bdev->blockcnt) {
    3359           0 :                 SPDK_ERRLOG("Data offset %lu exceeds base bdev capacity %lu on bdev '%s'\n",
    3360             :                             base_info->data_offset, bdev->blockcnt, base_info->name);
    3361           0 :                 rc = -EINVAL;
    3362           0 :                 goto out;
    3363             :         }
    3364             : 
    3365         510 :         if (base_info->data_size == 0) {
    3366         510 :                 base_info->data_size = bdev->blockcnt - base_info->data_offset;
    3367           0 :         } else if (base_info->data_offset + base_info->data_size > bdev->blockcnt) {
    3368           0 :                 SPDK_ERRLOG("Data offset and size exceeds base bdev capacity %lu on bdev '%s'\n",
    3369             :                             bdev->blockcnt, base_info->name);
    3370           0 :                 rc = -EINVAL;
    3371           0 :                 goto out;
    3372             :         }
    3373             : 
    3374         510 :         if (!raid_bdev->module->dif_supported && spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
    3375           0 :                 SPDK_ERRLOG("Base bdev '%s' has DIF or DIX enabled - unsupported RAID configuration\n",
    3376             :                             bdev->name);
    3377           0 :                 rc = -EINVAL;
    3378           0 :                 goto out;
    3379             :         }
    3380             : 
    3381             :         /*
    3382             :          * Set the raid bdev properties if this is the first base bdev configured,
    3383             :          * otherwise - verify. Assumption is that all the base bdevs for any raid bdev should
    3384             :          * have the same blocklen and metadata format.
    3385             :          */
    3386         510 :         if (raid_bdev->bdev.blocklen == 0) {
    3387          16 :                 raid_bdev->bdev.blocklen = bdev->blocklen;
    3388          16 :                 raid_bdev->bdev.md_len = spdk_bdev_get_md_size(bdev);
    3389          16 :                 raid_bdev->bdev.md_interleave = spdk_bdev_is_md_interleaved(bdev);
    3390          16 :                 raid_bdev->bdev.dif_type = spdk_bdev_get_dif_type(bdev);
    3391          16 :                 raid_bdev->bdev.dif_check_flags = bdev->dif_check_flags;
    3392          16 :                 raid_bdev->bdev.dif_is_head_of_md = spdk_bdev_is_dif_head_of_md(bdev);
    3393          16 :                 raid_bdev->bdev.dif_pi_format = bdev->dif_pi_format;
    3394             :         } else {
    3395         494 :                 if (raid_bdev->bdev.blocklen != bdev->blocklen) {
    3396           0 :                         SPDK_ERRLOG("Raid bdev '%s' blocklen %u differs from base bdev '%s' blocklen %u\n",
    3397             :                                     raid_bdev->bdev.name, raid_bdev->bdev.blocklen, bdev->name, bdev->blocklen);
    3398           0 :                         rc = -EINVAL;
    3399           0 :                         goto out;
    3400             :                 }
    3401             : 
    3402         988 :                 if (raid_bdev->bdev.md_len != spdk_bdev_get_md_size(bdev) ||
    3403         988 :                     raid_bdev->bdev.md_interleave != spdk_bdev_is_md_interleaved(bdev) ||
    3404         494 :                     raid_bdev->bdev.dif_type != spdk_bdev_get_dif_type(bdev) ||
    3405         988 :                     raid_bdev->bdev.dif_check_flags != bdev->dif_check_flags ||
    3406         494 :                     raid_bdev->bdev.dif_is_head_of_md != spdk_bdev_is_dif_head_of_md(bdev) ||
    3407         494 :                     raid_bdev->bdev.dif_pi_format != bdev->dif_pi_format) {
    3408           0 :                         SPDK_ERRLOG("Raid bdev '%s' has different metadata format than base bdev '%s'\n",
    3409             :                                     raid_bdev->bdev.name, bdev->name);
    3410           0 :                         rc = -EINVAL;
    3411           0 :                         goto out;
    3412             :                 }
    3413             :         }
    3414             : 
    3415         510 :         assert(base_info->configure_cb == NULL);
    3416         510 :         base_info->configure_cb = cb_fn;
    3417         510 :         base_info->configure_cb_ctx = cb_ctx;
    3418             : 
    3419         510 :         if (existing) {
    3420           0 :                 raid_bdev_configure_base_bdev_cont(base_info);
    3421             :         } else {
    3422             :                 /* check for existing superblock when using a new bdev */
    3423         510 :                 rc = raid_bdev_load_base_bdev_superblock(desc, base_info->app_thread_ch,
    3424             :                                 raid_bdev_configure_base_bdev_check_sb_cb, base_info);
    3425         510 :                 if (rc) {
    3426           0 :                         SPDK_ERRLOG("Failed to read bdev %s superblock: %s\n",
    3427             :                                     bdev->name, spdk_strerror(-rc));
    3428             :                 }
    3429             :         }
    3430         510 : out:
    3431         510 :         if (rc != 0) {
    3432           0 :                 base_info->configure_cb = NULL;
    3433           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    3434             :         }
    3435         510 :         return rc;
    3436             : }
    3437             : 
    3438             : int
    3439         513 : raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, const char *name,
    3440             :                         raid_base_bdev_cb cb_fn, void *cb_ctx)
    3441             : {
    3442         513 :         struct raid_base_bdev_info *base_info = NULL, *iter;
    3443             :         int rc;
    3444             : 
    3445         513 :         assert(name != NULL);
    3446         513 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3447             : 
    3448         513 :         if (raid_bdev->process != NULL) {
    3449           0 :                 SPDK_ERRLOG("raid bdev '%s' is in process\n",
    3450             :                             raid_bdev->bdev.name);
    3451           0 :                 return -EPERM;
    3452             :         }
    3453             : 
    3454         513 :         if (raid_bdev->state == RAID_BDEV_STATE_CONFIGURING) {
    3455         513 :                 struct spdk_bdev *bdev = spdk_bdev_get_by_name(name);
    3456             : 
    3457         513 :                 if (bdev != NULL) {
    3458       16896 :                         RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3459       24863 :                                 if (iter->name == NULL &&
    3460        8479 :                                     spdk_uuid_compare(&bdev->uuid, &iter->uuid) == 0) {
    3461           0 :                                         base_info = iter;
    3462           0 :                                         break;
    3463             :                                 }
    3464             :                         }
    3465             :                 }
    3466             :         }
    3467             : 
    3468         513 :         if (base_info == NULL || raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3469        8449 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3470        8449 :                         if (iter->name == NULL && spdk_uuid_is_null(&iter->uuid)) {
    3471         513 :                                 base_info = iter;
    3472         513 :                                 break;
    3473             :                         }
    3474             :                 }
    3475             :         }
    3476             : 
    3477         513 :         if (base_info == NULL) {
    3478           0 :                 SPDK_ERRLOG("no empty slot found in raid bdev '%s' for new base bdev '%s'\n",
    3479             :                             raid_bdev->bdev.name, name);
    3480           0 :                 return -EINVAL;
    3481             :         }
    3482             : 
    3483         513 :         assert(base_info->is_configured == false);
    3484             : 
    3485         513 :         if (raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3486           0 :                 assert(base_info->data_size != 0);
    3487           0 :                 assert(base_info->desc == NULL);
    3488             :         }
    3489             : 
    3490         513 :         base_info->name = strdup(name);
    3491         513 :         if (base_info->name == NULL) {
    3492           0 :                 return -ENOMEM;
    3493             :         }
    3494             : 
    3495         513 :         rc = raid_bdev_configure_base_bdev(base_info, false, cb_fn, cb_ctx);
    3496         513 :         if (rc != 0 && (rc != -ENODEV || raid_bdev->state != RAID_BDEV_STATE_CONFIGURING)) {
    3497           2 :                 SPDK_ERRLOG("base bdev '%s' configure failed: %s\n", name, spdk_strerror(-rc));
    3498           2 :                 free(base_info->name);
    3499           2 :                 base_info->name = NULL;
    3500             :         }
    3501             : 
    3502         513 :         return rc;
    3503             : }
    3504             : 
    3505             : static int
    3506           0 : raid_bdev_create_from_sb(const struct raid_bdev_superblock *sb, struct raid_bdev **raid_bdev_out)
    3507             : {
    3508           0 :         struct raid_bdev *raid_bdev;
    3509             :         uint8_t i;
    3510             :         int rc;
    3511             : 
    3512           0 :         rc = _raid_bdev_create(sb->name, (sb->strip_size * sb->block_size) / 1024, sb->num_base_bdevs,
    3513           0 :                                sb->level, true, &sb->uuid, &raid_bdev);
    3514           0 :         if (rc != 0) {
    3515           0 :                 return rc;
    3516             :         }
    3517             : 
    3518           0 :         rc = raid_bdev_alloc_superblock(raid_bdev, sb->block_size);
    3519           0 :         if (rc != 0) {
    3520           0 :                 raid_bdev_free(raid_bdev);
    3521           0 :                 return rc;
    3522             :         }
    3523             : 
    3524           0 :         assert(sb->length <= RAID_BDEV_SB_MAX_LENGTH);
    3525           0 :         memcpy(raid_bdev->sb, sb, sb->length);
    3526             : 
    3527           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    3528           0 :                 const struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    3529           0 :                 struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    3530             : 
    3531           0 :                 if (sb_base_bdev->state == RAID_SB_BASE_BDEV_CONFIGURED) {
    3532           0 :                         spdk_uuid_copy(&base_info->uuid, &sb_base_bdev->uuid);
    3533           0 :                         raid_bdev->num_base_bdevs_operational++;
    3534             :                 }
    3535             : 
    3536           0 :                 base_info->data_offset = sb_base_bdev->data_offset;
    3537           0 :                 base_info->data_size = sb_base_bdev->data_size;
    3538             :         }
    3539             : 
    3540           0 :         *raid_bdev_out = raid_bdev;
    3541           0 :         return 0;
    3542             : }
    3543             : 
    3544             : static void
    3545           0 : raid_bdev_examine_no_sb(struct spdk_bdev *bdev)
    3546             : {
    3547             :         struct raid_bdev *raid_bdev;
    3548             :         struct raid_base_bdev_info *base_info;
    3549             : 
    3550           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    3551           0 :                 if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING || raid_bdev->sb != NULL) {
    3552           0 :                         continue;
    3553             :                 }
    3554           0 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    3555           0 :                         if (base_info->desc == NULL &&
    3556           0 :                             ((base_info->name != NULL && strcmp(bdev->name, base_info->name) == 0) ||
    3557           0 :                              spdk_uuid_compare(&base_info->uuid, &bdev->uuid) == 0)) {
    3558           0 :                                 raid_bdev_configure_base_bdev(base_info, true, NULL, NULL);
    3559           0 :                                 break;
    3560             :                         }
    3561             :                 }
    3562             :         }
    3563           0 : }
    3564             : 
    3565             : struct raid_bdev_examine_others_ctx {
    3566             :         struct spdk_uuid raid_bdev_uuid;
    3567             :         uint8_t current_base_bdev_idx;
    3568             :         raid_base_bdev_cb cb_fn;
    3569             :         void *cb_ctx;
    3570             : };
    3571             : 
    3572             : static void
    3573           0 : raid_bdev_examine_others_done(void *_ctx, int status)
    3574             : {
    3575           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3576             : 
    3577           0 :         if (ctx->cb_fn != NULL) {
    3578           0 :                 ctx->cb_fn(ctx->cb_ctx, status);
    3579             :         }
    3580           0 :         free(ctx);
    3581           0 : }
    3582             : 
    3583             : typedef void (*raid_bdev_examine_load_sb_cb)(struct spdk_bdev *bdev,
    3584             :                 const struct raid_bdev_superblock *sb, int status, void *ctx);
    3585             : static int raid_bdev_examine_load_sb(const char *bdev_name, raid_bdev_examine_load_sb_cb cb,
    3586             :                                      void *cb_ctx);
    3587             : static void raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3588             :                                  raid_base_bdev_cb cb_fn, void *cb_ctx);
    3589             : static void raid_bdev_examine_others(void *_ctx, int status);
    3590             : 
    3591             : static void
    3592           0 : raid_bdev_examine_others_load_cb(struct spdk_bdev *bdev, const struct raid_bdev_superblock *sb,
    3593             :                                  int status, void *_ctx)
    3594             : {
    3595           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3596             : 
    3597           0 :         if (status != 0) {
    3598           0 :                 raid_bdev_examine_others_done(ctx, status);
    3599           0 :                 return;
    3600             :         }
    3601             : 
    3602           0 :         raid_bdev_examine_sb(sb, bdev, raid_bdev_examine_others, ctx);
    3603             : }
    3604             : 
    3605             : static void
    3606           0 : raid_bdev_examine_others(void *_ctx, int status)
    3607             : {
    3608           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3609             :         struct raid_bdev *raid_bdev;
    3610             :         struct raid_base_bdev_info *base_info;
    3611           0 :         char uuid_str[SPDK_UUID_STRING_LEN];
    3612             : 
    3613           0 :         if (status != 0 && status != -EEXIST) {
    3614           0 :                 goto out;
    3615             :         }
    3616             : 
    3617           0 :         raid_bdev = raid_bdev_find_by_uuid(&ctx->raid_bdev_uuid);
    3618           0 :         if (raid_bdev == NULL) {
    3619           0 :                 status = -ENODEV;
    3620           0 :                 goto out;
    3621             :         }
    3622             : 
    3623           0 :         for (base_info = &raid_bdev->base_bdev_info[ctx->current_base_bdev_idx];
    3624           0 :              base_info < &raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs];
    3625           0 :              base_info++) {
    3626           0 :                 if (base_info->is_configured || spdk_uuid_is_null(&base_info->uuid)) {
    3627           0 :                         continue;
    3628             :                 }
    3629             : 
    3630           0 :                 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &base_info->uuid);
    3631             : 
    3632           0 :                 if (spdk_bdev_get_by_name(uuid_str) == NULL) {
    3633           0 :                         continue;
    3634             :                 }
    3635             : 
    3636           0 :                 ctx->current_base_bdev_idx = raid_bdev_base_bdev_slot(base_info);
    3637             : 
    3638           0 :                 status = raid_bdev_examine_load_sb(uuid_str, raid_bdev_examine_others_load_cb, ctx);
    3639           0 :                 if (status != 0) {
    3640           0 :                         continue;
    3641             :                 }
    3642           0 :                 return;
    3643             :         }
    3644           0 : out:
    3645           0 :         raid_bdev_examine_others_done(ctx, status);
    3646             : }
    3647             : 
    3648             : static void
    3649           0 : raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3650             :                      raid_base_bdev_cb cb_fn, void *cb_ctx)
    3651             : {
    3652           0 :         const struct raid_bdev_sb_base_bdev *sb_base_bdev = NULL;
    3653           0 :         struct raid_bdev *raid_bdev;
    3654             :         struct raid_base_bdev_info *iter, *base_info;
    3655             :         uint8_t i;
    3656             :         int rc;
    3657             : 
    3658           0 :         if (sb->block_size != spdk_bdev_get_data_block_size(bdev)) {
    3659           0 :                 SPDK_WARNLOG("Bdev %s block size (%u) does not match the value in superblock (%u)\n",
    3660             :                              bdev->name, sb->block_size, spdk_bdev_get_data_block_size(bdev));
    3661           0 :                 rc = -EINVAL;
    3662           0 :                 goto out;
    3663             :         }
    3664             : 
    3665           0 :         if (spdk_uuid_is_null(&sb->uuid)) {
    3666           0 :                 SPDK_WARNLOG("NULL raid bdev UUID in superblock on bdev %s\n", bdev->name);
    3667           0 :                 rc = -EINVAL;
    3668           0 :                 goto out;
    3669             :         }
    3670             : 
    3671           0 :         raid_bdev = raid_bdev_find_by_uuid(&sb->uuid);
    3672             : 
    3673           0 :         if (raid_bdev) {
    3674           0 :                 if (sb->seq_number > raid_bdev->sb->seq_number) {
    3675           0 :                         SPDK_DEBUGLOG(bdev_raid,
    3676             :                                       "raid superblock seq_number on bdev %s (%lu) greater than existing raid bdev %s (%lu)\n",
    3677             :                                       bdev->name, sb->seq_number, raid_bdev->bdev.name, raid_bdev->sb->seq_number);
    3678             : 
    3679           0 :                         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
    3680           0 :                                 SPDK_WARNLOG("Newer version of raid bdev %s superblock found on bdev %s but raid bdev is not in configuring state.\n",
    3681             :                                              raid_bdev->bdev.name, bdev->name);
    3682           0 :                                 rc = -EBUSY;
    3683           0 :                                 goto out;
    3684             :                         }
    3685             : 
    3686             :                         /* remove and then recreate the raid bdev using the newer superblock */
    3687           0 :                         raid_bdev_delete(raid_bdev, NULL, NULL);
    3688           0 :                         raid_bdev = NULL;
    3689           0 :                 } else if (sb->seq_number < raid_bdev->sb->seq_number) {
    3690           0 :                         SPDK_DEBUGLOG(bdev_raid,
    3691             :                                       "raid superblock seq_number on bdev %s (%lu) smaller than existing raid bdev %s (%lu)\n",
    3692             :                                       bdev->name, sb->seq_number, raid_bdev->bdev.name, raid_bdev->sb->seq_number);
    3693             :                         /* use the current raid bdev superblock */
    3694           0 :                         sb = raid_bdev->sb;
    3695             :                 }
    3696             :         }
    3697             : 
    3698           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    3699           0 :                 sb_base_bdev = &sb->base_bdevs[i];
    3700             : 
    3701           0 :                 assert(spdk_uuid_is_null(&sb_base_bdev->uuid) == false);
    3702             : 
    3703           0 :                 if (spdk_uuid_compare(&sb_base_bdev->uuid, spdk_bdev_get_uuid(bdev)) == 0) {
    3704           0 :                         break;
    3705             :                 }
    3706             :         }
    3707             : 
    3708           0 :         if (i == sb->base_bdevs_size) {
    3709           0 :                 SPDK_DEBUGLOG(bdev_raid, "raid superblock does not contain this bdev's uuid\n");
    3710           0 :                 rc = -EINVAL;
    3711           0 :                 goto out;
    3712             :         }
    3713             : 
    3714           0 :         if (!raid_bdev) {
    3715             :                 struct raid_bdev_examine_others_ctx *ctx;
    3716             : 
    3717           0 :                 ctx = calloc(1, sizeof(*ctx));
    3718           0 :                 if (ctx == NULL) {
    3719           0 :                         rc = -ENOMEM;
    3720           0 :                         goto out;
    3721             :                 }
    3722             : 
    3723           0 :                 rc = raid_bdev_create_from_sb(sb, &raid_bdev);
    3724           0 :                 if (rc != 0) {
    3725           0 :                         SPDK_ERRLOG("Failed to create raid bdev %s: %s\n",
    3726             :                                     sb->name, spdk_strerror(-rc));
    3727           0 :                         free(ctx);
    3728           0 :                         goto out;
    3729             :                 }
    3730             : 
    3731             :                 /* after this base bdev is configured, examine other base bdevs that may be present */
    3732           0 :                 spdk_uuid_copy(&ctx->raid_bdev_uuid, &sb->uuid);
    3733           0 :                 ctx->cb_fn = cb_fn;
    3734           0 :                 ctx->cb_ctx = cb_ctx;
    3735             : 
    3736           0 :                 cb_fn = raid_bdev_examine_others;
    3737           0 :                 cb_ctx = ctx;
    3738             :         }
    3739             : 
    3740           0 :         if (raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3741           0 :                 assert(sb_base_bdev->slot < raid_bdev->num_base_bdevs);
    3742           0 :                 base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    3743           0 :                 assert(base_info->is_configured == false);
    3744           0 :                 assert(sb_base_bdev->state == RAID_SB_BASE_BDEV_MISSING ||
    3745             :                        sb_base_bdev->state == RAID_SB_BASE_BDEV_FAILED);
    3746           0 :                 assert(spdk_uuid_is_null(&base_info->uuid));
    3747           0 :                 spdk_uuid_copy(&base_info->uuid, &sb_base_bdev->uuid);
    3748           0 :                 SPDK_NOTICELOG("Re-adding bdev %s to raid bdev %s.\n", bdev->name, raid_bdev->bdev.name);
    3749           0 :                 rc = raid_bdev_configure_base_bdev(base_info, true, cb_fn, cb_ctx);
    3750           0 :                 if (rc != 0) {
    3751           0 :                         SPDK_ERRLOG("Failed to configure bdev %s as base bdev of raid %s: %s\n",
    3752             :                                     bdev->name, raid_bdev->bdev.name, spdk_strerror(-rc));
    3753             :                 }
    3754           0 :                 goto out;
    3755             :         }
    3756             : 
    3757           0 :         if (sb_base_bdev->state != RAID_SB_BASE_BDEV_CONFIGURED) {
    3758           0 :                 SPDK_NOTICELOG("Bdev %s is not an active member of raid bdev %s. Ignoring.\n",
    3759             :                                bdev->name, raid_bdev->bdev.name);
    3760           0 :                 rc = -EINVAL;
    3761           0 :                 goto out;
    3762             :         }
    3763             : 
    3764           0 :         base_info = NULL;
    3765           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3766           0 :                 if (spdk_uuid_compare(&iter->uuid, spdk_bdev_get_uuid(bdev)) == 0) {
    3767           0 :                         base_info = iter;
    3768           0 :                         break;
    3769             :                 }
    3770             :         }
    3771             : 
    3772           0 :         if (base_info == NULL) {
    3773           0 :                 SPDK_ERRLOG("Bdev %s is not a member of raid bdev %s\n",
    3774             :                             bdev->name, raid_bdev->bdev.name);
    3775           0 :                 rc = -EINVAL;
    3776           0 :                 goto out;
    3777             :         }
    3778             : 
    3779           0 :         if (base_info->is_configured) {
    3780           0 :                 rc = -EEXIST;
    3781           0 :                 goto out;
    3782             :         }
    3783             : 
    3784           0 :         rc = raid_bdev_configure_base_bdev(base_info, true, cb_fn, cb_ctx);
    3785           0 :         if (rc != 0) {
    3786           0 :                 SPDK_ERRLOG("Failed to configure bdev %s as base bdev of raid %s: %s\n",
    3787             :                             bdev->name, raid_bdev->bdev.name, spdk_strerror(-rc));
    3788             :         }
    3789           0 : out:
    3790           0 :         if (rc != 0 && cb_fn != 0) {
    3791           0 :                 cb_fn(cb_ctx, rc);
    3792             :         }
    3793           0 : }
    3794             : 
    3795             : struct raid_bdev_examine_ctx {
    3796             :         struct spdk_bdev_desc *desc;
    3797             :         struct spdk_io_channel *ch;
    3798             :         raid_bdev_examine_load_sb_cb cb;
    3799             :         void *cb_ctx;
    3800             : };
    3801             : 
    3802             : static void
    3803           0 : raid_bdev_examine_ctx_free(struct raid_bdev_examine_ctx *ctx)
    3804             : {
    3805           0 :         if (!ctx) {
    3806           0 :                 return;
    3807             :         }
    3808             : 
    3809           0 :         if (ctx->ch) {
    3810           0 :                 spdk_put_io_channel(ctx->ch);
    3811             :         }
    3812             : 
    3813           0 :         if (ctx->desc) {
    3814           0 :                 spdk_bdev_close(ctx->desc);
    3815             :         }
    3816             : 
    3817           0 :         free(ctx);
    3818             : }
    3819             : 
    3820             : static void
    3821           0 : raid_bdev_examine_load_sb_done(const struct raid_bdev_superblock *sb, int status, void *_ctx)
    3822             : {
    3823           0 :         struct raid_bdev_examine_ctx *ctx = _ctx;
    3824           0 :         struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(ctx->desc);
    3825             : 
    3826           0 :         ctx->cb(bdev, sb, status, ctx->cb_ctx);
    3827             : 
    3828           0 :         raid_bdev_examine_ctx_free(ctx);
    3829           0 : }
    3830             : 
    3831             : static void
    3832           0 : raid_bdev_examine_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx)
    3833             : {
    3834           0 : }
    3835             : 
    3836             : static int
    3837           0 : raid_bdev_examine_load_sb(const char *bdev_name, raid_bdev_examine_load_sb_cb cb, void *cb_ctx)
    3838             : {
    3839             :         struct raid_bdev_examine_ctx *ctx;
    3840             :         int rc;
    3841             : 
    3842           0 :         assert(cb != NULL);
    3843             : 
    3844           0 :         ctx = calloc(1, sizeof(*ctx));
    3845           0 :         if (!ctx) {
    3846           0 :                 return -ENOMEM;
    3847             :         }
    3848             : 
    3849           0 :         rc = spdk_bdev_open_ext(bdev_name, false, raid_bdev_examine_event_cb, NULL, &ctx->desc);
    3850           0 :         if (rc) {
    3851           0 :                 SPDK_ERRLOG("Failed to open bdev %s: %s\n", bdev_name, spdk_strerror(-rc));
    3852           0 :                 goto err;
    3853             :         }
    3854             : 
    3855           0 :         ctx->ch = spdk_bdev_get_io_channel(ctx->desc);
    3856           0 :         if (!ctx->ch) {
    3857           0 :                 SPDK_ERRLOG("Failed to get io channel for bdev %s\n", bdev_name);
    3858           0 :                 rc = -ENOMEM;
    3859           0 :                 goto err;
    3860             :         }
    3861             : 
    3862           0 :         ctx->cb = cb;
    3863           0 :         ctx->cb_ctx = cb_ctx;
    3864             : 
    3865           0 :         rc = raid_bdev_load_base_bdev_superblock(ctx->desc, ctx->ch, raid_bdev_examine_load_sb_done, ctx);
    3866           0 :         if (rc) {
    3867           0 :                 SPDK_ERRLOG("Failed to read bdev %s superblock: %s\n",
    3868             :                             bdev_name, spdk_strerror(-rc));
    3869           0 :                 goto err;
    3870             :         }
    3871             : 
    3872           0 :         return 0;
    3873           0 : err:
    3874           0 :         raid_bdev_examine_ctx_free(ctx);
    3875           0 :         return rc;
    3876             : }
    3877             : 
    3878             : static void
    3879           0 : raid_bdev_examine_done(void *ctx, int status)
    3880             : {
    3881           0 :         struct spdk_bdev *bdev = ctx;
    3882             : 
    3883           0 :         if (status != 0) {
    3884           0 :                 SPDK_ERRLOG("Failed to examine bdev %s: %s\n",
    3885             :                             bdev->name, spdk_strerror(-status));
    3886             :         }
    3887           0 :         spdk_bdev_module_examine_done(&g_raid_if);
    3888           0 : }
    3889             : 
    3890             : static void
    3891           0 : raid_bdev_examine_cont(struct spdk_bdev *bdev, const struct raid_bdev_superblock *sb, int status,
    3892             :                        void *ctx)
    3893             : {
    3894           0 :         switch (status) {
    3895           0 :         case 0:
    3896             :                 /* valid superblock found */
    3897           0 :                 SPDK_DEBUGLOG(bdev_raid, "raid superblock found on bdev %s\n", bdev->name);
    3898           0 :                 raid_bdev_examine_sb(sb, bdev, raid_bdev_examine_done, bdev);
    3899           0 :                 return;
    3900           0 :         case -EINVAL:
    3901             :                 /* no valid superblock, check if it can be claimed anyway */
    3902           0 :                 raid_bdev_examine_no_sb(bdev);
    3903           0 :                 status = 0;
    3904           0 :                 break;
    3905             :         }
    3906             : 
    3907           0 :         raid_bdev_examine_done(bdev, status);
    3908             : }
    3909             : 
    3910             : /*
    3911             :  * brief:
    3912             :  * raid_bdev_examine function is the examine function call by the below layers
    3913             :  * like bdev_nvme layer. This function will check if this base bdev can be
    3914             :  * claimed by this raid bdev or not.
    3915             :  * params:
    3916             :  * bdev - pointer to base bdev
    3917             :  * returns:
    3918             :  * none
    3919             :  */
    3920             : static void
    3921           0 : raid_bdev_examine(struct spdk_bdev *bdev)
    3922             : {
    3923           0 :         int rc = 0;
    3924             : 
    3925           0 :         if (raid_bdev_find_base_info_by_bdev(bdev) != NULL) {
    3926           0 :                 goto done;
    3927             :         }
    3928             : 
    3929           0 :         if (spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
    3930           0 :                 raid_bdev_examine_no_sb(bdev);
    3931           0 :                 goto done;
    3932             :         }
    3933             : 
    3934           0 :         rc = raid_bdev_examine_load_sb(bdev->name, raid_bdev_examine_cont, NULL);
    3935           0 :         if (rc != 0) {
    3936           0 :                 goto done;
    3937             :         }
    3938             : 
    3939           0 :         return;
    3940           0 : done:
    3941           0 :         raid_bdev_examine_done(bdev, rc);
    3942             : }
    3943             : 
    3944             : /* Log component for bdev raid bdev module */
    3945           1 : SPDK_LOG_REGISTER_COMPONENT(bdev_raid)
    3946             : 
    3947             : static void
    3948           0 : bdev_raid_trace(void)
    3949             : {
    3950           0 :         struct spdk_trace_tpoint_opts opts[] = {
    3951             :                 {
    3952             :                         "BDEV_RAID_IO_START", TRACE_BDEV_RAID_IO_START,
    3953             :                         OWNER_TYPE_NONE, OBJECT_BDEV_RAID_IO, 1,
    3954             :                         {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
    3955             :                 },
    3956             :                 {
    3957             :                         "BDEV_RAID_IO_DONE", TRACE_BDEV_RAID_IO_DONE,
    3958             :                         OWNER_TYPE_NONE, OBJECT_BDEV_RAID_IO, 0,
    3959             :                         {{ "ctx", SPDK_TRACE_ARG_TYPE_PTR, 8 }}
    3960             :                 }
    3961             :         };
    3962             : 
    3963             : 
    3964           0 :         spdk_trace_register_object(OBJECT_BDEV_RAID_IO, 'R');
    3965           0 :         spdk_trace_register_description_ext(opts, SPDK_COUNTOF(opts));
    3966           0 :         spdk_trace_tpoint_register_relation(TRACE_BDEV_IO_START, OBJECT_BDEV_RAID_IO, 1);
    3967           0 :         spdk_trace_tpoint_register_relation(TRACE_BDEV_IO_DONE, OBJECT_BDEV_RAID_IO, 0);
    3968           0 : }
    3969           1 : SPDK_TRACE_REGISTER_FN(bdev_raid_trace, "bdev_raid", TRACE_GROUP_BDEV_RAID)

Generated by: LCOV version 1.15