LCOV - code coverage report
Current view: top level - module/bdev/raid - bdev_raid.c (source / functions) Hit Total Coverage
Test: ut_cov_unit.info Lines: 834 1787 46.7 %
Date: 2024-08-13 14:04:00 Functions: 80 142 56.3 %

          Line data    Source code
       1             : /*   SPDX-License-Identifier: BSD-3-Clause
       2             :  *   Copyright (C) 2018 Intel Corporation.
       3             :  *   All rights reserved.
       4             :  *   Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
       5             :  */
       6             : 
       7             : #include "bdev_raid.h"
       8             : #include "spdk/env.h"
       9             : #include "spdk/thread.h"
      10             : #include "spdk/log.h"
      11             : #include "spdk/string.h"
      12             : #include "spdk/util.h"
      13             : #include "spdk/json.h"
      14             : #include "spdk/likely.h"
      15             : 
      16             : #define RAID_OFFSET_BLOCKS_INVALID      UINT64_MAX
      17             : #define RAID_BDEV_PROCESS_MAX_QD        16
      18             : 
      19             : #define RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT        1024
      20             : #define RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT  0
      21             : 
      22             : static bool g_shutdown_started = false;
      23             : 
      24             : /* List of all raid bdevs */
      25             : struct raid_all_tailq g_raid_bdev_list = TAILQ_HEAD_INITIALIZER(g_raid_bdev_list);
      26             : 
      27             : static TAILQ_HEAD(, raid_bdev_module) g_raid_modules = TAILQ_HEAD_INITIALIZER(g_raid_modules);
      28             : 
      29             : /*
      30             :  * raid_bdev_io_channel is the context of spdk_io_channel for raid bdev device. It
      31             :  * contains the relationship of raid bdev io channel with base bdev io channels.
      32             :  */
      33             : struct raid_bdev_io_channel {
      34             :         /* Array of IO channels of base bdevs */
      35             :         struct spdk_io_channel  **base_channel;
      36             : 
      37             :         /* Private raid module IO channel */
      38             :         struct spdk_io_channel  *module_channel;
      39             : 
      40             :         /* Background process data */
      41             :         struct {
      42             :                 uint64_t offset;
      43             :                 struct spdk_io_channel *target_ch;
      44             :                 struct raid_bdev_io_channel *ch_processed;
      45             :         } process;
      46             : };
      47             : 
      48             : enum raid_bdev_process_state {
      49             :         RAID_PROCESS_STATE_INIT,
      50             :         RAID_PROCESS_STATE_RUNNING,
      51             :         RAID_PROCESS_STATE_STOPPING,
      52             :         RAID_PROCESS_STATE_STOPPED,
      53             : };
      54             : 
      55             : struct raid_process_qos {
      56             :         bool enable_qos;
      57             :         uint64_t last_tsc;
      58             :         double bytes_per_tsc;
      59             :         double bytes_available;
      60             :         double bytes_max;
      61             :         struct spdk_poller *process_continue_poller;
      62             : };
      63             : 
      64             : struct raid_bdev_process {
      65             :         struct raid_bdev                *raid_bdev;
      66             :         enum raid_process_type          type;
      67             :         enum raid_bdev_process_state    state;
      68             :         struct spdk_thread              *thread;
      69             :         struct raid_bdev_io_channel     *raid_ch;
      70             :         TAILQ_HEAD(, raid_bdev_process_request) requests;
      71             :         uint64_t                        max_window_size;
      72             :         uint64_t                        window_size;
      73             :         uint64_t                        window_remaining;
      74             :         int                             window_status;
      75             :         uint64_t                        window_offset;
      76             :         bool                            window_range_locked;
      77             :         struct raid_base_bdev_info      *target;
      78             :         int                             status;
      79             :         TAILQ_HEAD(, raid_process_finish_action) finish_actions;
      80             :         struct raid_process_qos         qos;
      81             : };
      82             : 
      83             : struct raid_process_finish_action {
      84             :         spdk_msg_fn cb;
      85             :         void *cb_ctx;
      86             :         TAILQ_ENTRY(raid_process_finish_action) link;
      87             : };
      88             : 
      89             : static struct spdk_raid_bdev_opts g_opts = {
      90             :         .process_window_size_kb = RAID_BDEV_PROCESS_WINDOW_SIZE_KB_DEFAULT,
      91             :         .process_max_bandwidth_mb_sec = RAID_BDEV_PROCESS_MAX_BANDWIDTH_MB_SEC_DEFAULT,
      92             : };
      93             : 
      94             : void
      95           0 : raid_bdev_get_opts(struct spdk_raid_bdev_opts *opts)
      96             : {
      97           0 :         *opts = g_opts;
      98           0 : }
      99             : 
     100             : int
     101           1 : raid_bdev_set_opts(const struct spdk_raid_bdev_opts *opts)
     102             : {
     103           1 :         if (opts->process_window_size_kb == 0) {
     104           0 :                 return -EINVAL;
     105             :         }
     106             : 
     107           1 :         g_opts = *opts;
     108             : 
     109           1 :         return 0;
     110             : }
     111             : 
     112             : static struct raid_bdev_module *
     113          19 : raid_bdev_module_find(enum raid_level level)
     114             : {
     115             :         struct raid_bdev_module *raid_module;
     116             : 
     117          20 :         TAILQ_FOREACH(raid_module, &g_raid_modules, link) {
     118          18 :                 if (raid_module->level == level) {
     119          17 :                         return raid_module;
     120             :                 }
     121             :         }
     122             : 
     123           2 :         return NULL;
     124             : }
     125             : 
     126             : void
     127           1 : raid_bdev_module_list_add(struct raid_bdev_module *raid_module)
     128             : {
     129           1 :         if (raid_bdev_module_find(raid_module->level) != NULL) {
     130           0 :                 SPDK_ERRLOG("module for raid level '%s' already registered.\n",
     131             :                             raid_bdev_level_to_str(raid_module->level));
     132           0 :                 assert(false);
     133             :         } else {
     134           1 :                 TAILQ_INSERT_TAIL(&g_raid_modules, raid_module, link);
     135             :         }
     136           1 : }
     137             : 
     138             : struct spdk_io_channel *
     139           0 : raid_bdev_channel_get_base_channel(struct raid_bdev_io_channel *raid_ch, uint8_t idx)
     140             : {
     141           0 :         return raid_ch->base_channel[idx];
     142             : }
     143             : 
     144             : void *
     145           0 : raid_bdev_channel_get_module_ctx(struct raid_bdev_io_channel *raid_ch)
     146             : {
     147           0 :         assert(raid_ch->module_channel != NULL);
     148             : 
     149           0 :         return spdk_io_channel_get_ctx(raid_ch->module_channel);
     150             : }
     151             : 
     152             : struct raid_base_bdev_info *
     153           0 : raid_bdev_channel_get_base_info(struct raid_bdev_io_channel *raid_ch, struct spdk_bdev *base_bdev)
     154             : {
     155           0 :         struct spdk_io_channel *ch = spdk_io_channel_from_ctx(raid_ch);
     156           0 :         struct raid_bdev *raid_bdev = spdk_io_channel_get_io_device(ch);
     157             :         uint8_t i;
     158             : 
     159           0 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     160           0 :                 struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[i];
     161             : 
     162           0 :                 if (base_info->is_configured &&
     163           0 :                     spdk_bdev_desc_get_bdev(base_info->desc) == base_bdev) {
     164           0 :                         return base_info;
     165             :                 }
     166             :         }
     167             : 
     168           0 :         return NULL;
     169             : }
     170             : 
     171             : /* Function declarations */
     172             : static void     raid_bdev_examine(struct spdk_bdev *bdev);
     173             : static int      raid_bdev_init(void);
     174             : static void     raid_bdev_deconfigure(struct raid_bdev *raid_bdev,
     175             :                                       raid_bdev_destruct_cb cb_fn, void *cb_arg);
     176             : 
     177             : static void
     178           7 : raid_bdev_ch_process_cleanup(struct raid_bdev_io_channel *raid_ch)
     179             : {
     180           7 :         raid_ch->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     181             : 
     182           7 :         if (raid_ch->process.target_ch != NULL) {
     183           1 :                 spdk_put_io_channel(raid_ch->process.target_ch);
     184           1 :                 raid_ch->process.target_ch = NULL;
     185             :         }
     186             : 
     187           7 :         if (raid_ch->process.ch_processed != NULL) {
     188           3 :                 free(raid_ch->process.ch_processed->base_channel);
     189           3 :                 free(raid_ch->process.ch_processed);
     190           3 :                 raid_ch->process.ch_processed = NULL;
     191             :         }
     192           7 : }
     193             : 
     194             : static int
     195           3 : raid_bdev_ch_process_setup(struct raid_bdev_io_channel *raid_ch, struct raid_bdev_process *process)
     196             : {
     197           3 :         struct raid_bdev *raid_bdev = process->raid_bdev;
     198             :         struct raid_bdev_io_channel *raid_ch_processed;
     199             :         struct raid_base_bdev_info *base_info;
     200             : 
     201           3 :         raid_ch->process.offset = process->window_offset;
     202             : 
     203             :         /* In the future we may have other types of processes which don't use a target bdev,
     204             :          * like data scrubbing or strip size migration. Until then, expect that there always is
     205             :          * a process target. */
     206           3 :         assert(process->target != NULL);
     207             : 
     208           3 :         raid_ch->process.target_ch = spdk_bdev_get_io_channel(process->target->desc);
     209           3 :         if (raid_ch->process.target_ch == NULL) {
     210           0 :                 goto err;
     211             :         }
     212             : 
     213           3 :         raid_ch_processed = calloc(1, sizeof(*raid_ch_processed));
     214           3 :         if (raid_ch_processed == NULL) {
     215           0 :                 goto err;
     216             :         }
     217           3 :         raid_ch->process.ch_processed = raid_ch_processed;
     218             : 
     219           3 :         raid_ch_processed->base_channel = calloc(raid_bdev->num_base_bdevs,
     220             :                                           sizeof(*raid_ch_processed->base_channel));
     221           3 :         if (raid_ch_processed->base_channel == NULL) {
     222           0 :                 goto err;
     223             :         }
     224             : 
     225          99 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     226          96 :                 uint8_t slot = raid_bdev_base_bdev_slot(base_info);
     227             : 
     228          96 :                 if (base_info != process->target) {
     229          93 :                         raid_ch_processed->base_channel[slot] = raid_ch->base_channel[slot];
     230             :                 } else {
     231           3 :                         raid_ch_processed->base_channel[slot] = raid_ch->process.target_ch;
     232             :                 }
     233             :         }
     234             : 
     235           3 :         raid_ch_processed->module_channel = raid_ch->module_channel;
     236           3 :         raid_ch_processed->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     237             : 
     238           3 :         return 0;
     239           0 : err:
     240           0 :         raid_bdev_ch_process_cleanup(raid_ch);
     241           0 :         return -ENOMEM;
     242             : }
     243             : 
     244             : /*
     245             :  * brief:
     246             :  * raid_bdev_create_cb function is a cb function for raid bdev which creates the
     247             :  * hierarchy from raid bdev to base bdev io channels. It will be called per core
     248             :  * params:
     249             :  * io_device - pointer to raid bdev io device represented by raid_bdev
     250             :  * ctx_buf - pointer to context buffer for raid bdev io channel
     251             :  * returns:
     252             :  * 0 - success
     253             :  * non zero - failure
     254             :  */
     255             : static int
     256           5 : raid_bdev_create_cb(void *io_device, void *ctx_buf)
     257             : {
     258           5 :         struct raid_bdev            *raid_bdev = io_device;
     259           5 :         struct raid_bdev_io_channel *raid_ch = ctx_buf;
     260             :         uint8_t i;
     261           5 :         int ret = -ENOMEM;
     262             : 
     263           5 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_create_cb, %p\n", raid_ch);
     264             : 
     265           5 :         assert(raid_bdev != NULL);
     266           5 :         assert(raid_bdev->state == RAID_BDEV_STATE_ONLINE);
     267             : 
     268           5 :         raid_ch->base_channel = calloc(raid_bdev->num_base_bdevs, sizeof(struct spdk_io_channel *));
     269           5 :         if (!raid_ch->base_channel) {
     270           0 :                 SPDK_ERRLOG("Unable to allocate base bdevs io channel\n");
     271           0 :                 return -ENOMEM;
     272             :         }
     273             : 
     274         165 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     275             :                 /*
     276             :                  * Get the spdk_io_channel for all the base bdevs. This is used during
     277             :                  * split logic to send the respective child bdev ios to respective base
     278             :                  * bdev io channel.
     279             :                  * Skip missing base bdevs and the process target, which should also be treated as
     280             :                  * missing until the process completes.
     281             :                  */
     282         160 :                 if (raid_bdev->base_bdev_info[i].is_configured == false ||
     283         160 :                     raid_bdev->base_bdev_info[i].is_process_target == true) {
     284           0 :                         continue;
     285             :                 }
     286         320 :                 raid_ch->base_channel[i] = spdk_bdev_get_io_channel(
     287         160 :                                                    raid_bdev->base_bdev_info[i].desc);
     288         160 :                 if (!raid_ch->base_channel[i]) {
     289           0 :                         SPDK_ERRLOG("Unable to create io channel for base bdev\n");
     290           0 :                         goto err;
     291             :                 }
     292             :         }
     293             : 
     294           5 :         if (raid_bdev->module->get_io_channel) {
     295           0 :                 raid_ch->module_channel = raid_bdev->module->get_io_channel(raid_bdev);
     296           0 :                 if (!raid_ch->module_channel) {
     297           0 :                         SPDK_ERRLOG("Unable to create io channel for raid module\n");
     298           0 :                         goto err;
     299             :                 }
     300             :         }
     301             : 
     302           5 :         if (raid_bdev->process != NULL) {
     303           3 :                 ret = raid_bdev_ch_process_setup(raid_ch, raid_bdev->process);
     304           3 :                 if (ret != 0) {
     305           0 :                         SPDK_ERRLOG("Failed to setup process io channel\n");
     306           0 :                         goto err;
     307             :                 }
     308             :         } else {
     309           2 :                 raid_ch->process.offset = RAID_OFFSET_BLOCKS_INVALID;
     310             :         }
     311             : 
     312           5 :         return 0;
     313           0 : err:
     314           0 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     315           0 :                 if (raid_ch->base_channel[i] != NULL) {
     316           0 :                         spdk_put_io_channel(raid_ch->base_channel[i]);
     317             :                 }
     318             :         }
     319           0 :         free(raid_ch->base_channel);
     320             : 
     321           0 :         raid_bdev_ch_process_cleanup(raid_ch);
     322             : 
     323           0 :         return ret;
     324             : }
     325             : 
     326             : /*
     327             :  * brief:
     328             :  * raid_bdev_destroy_cb function is a cb function for raid bdev which deletes the
     329             :  * hierarchy from raid bdev to base bdev io channels. It will be called per core
     330             :  * params:
     331             :  * io_device - pointer to raid bdev io device represented by raid_bdev
     332             :  * ctx_buf - pointer to context buffer for raid bdev io channel
     333             :  * returns:
     334             :  * none
     335             :  */
     336             : static void
     337           5 : raid_bdev_destroy_cb(void *io_device, void *ctx_buf)
     338             : {
     339           5 :         struct raid_bdev *raid_bdev = io_device;
     340           5 :         struct raid_bdev_io_channel *raid_ch = ctx_buf;
     341             :         uint8_t i;
     342             : 
     343           5 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_destroy_cb\n");
     344             : 
     345           5 :         assert(raid_ch != NULL);
     346           5 :         assert(raid_ch->base_channel);
     347             : 
     348           5 :         if (raid_ch->module_channel) {
     349           0 :                 spdk_put_io_channel(raid_ch->module_channel);
     350             :         }
     351             : 
     352         165 :         for (i = 0; i < raid_bdev->num_base_bdevs; i++) {
     353             :                 /* Free base bdev channels */
     354         160 :                 if (raid_ch->base_channel[i] != NULL) {
     355         160 :                         spdk_put_io_channel(raid_ch->base_channel[i]);
     356             :                 }
     357             :         }
     358           5 :         free(raid_ch->base_channel);
     359           5 :         raid_ch->base_channel = NULL;
     360             : 
     361           5 :         raid_bdev_ch_process_cleanup(raid_ch);
     362           5 : }
     363             : 
     364             : /*
     365             :  * brief:
     366             :  * raid_bdev_cleanup is used to cleanup raid_bdev related data
     367             :  * structures.
     368             :  * params:
     369             :  * raid_bdev - pointer to raid_bdev
     370             :  * returns:
     371             :  * none
     372             :  */
     373             : static void
     374          17 : raid_bdev_cleanup(struct raid_bdev *raid_bdev)
     375             : {
     376             :         struct raid_base_bdev_info *base_info;
     377             : 
     378          17 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_cleanup, %p name %s, state %s\n",
     379             :                       raid_bdev, raid_bdev->bdev.name, raid_bdev_state_to_str(raid_bdev->state));
     380          17 :         assert(raid_bdev->state != RAID_BDEV_STATE_ONLINE);
     381          17 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
     382             : 
     383         561 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     384         544 :                 assert(base_info->desc == NULL);
     385         544 :                 free(base_info->name);
     386             :         }
     387             : 
     388          17 :         TAILQ_REMOVE(&g_raid_bdev_list, raid_bdev, global_link);
     389          17 : }
     390             : 
     391             : static void
     392          17 : raid_bdev_free(struct raid_bdev *raid_bdev)
     393             : {
     394          17 :         raid_bdev_free_superblock(raid_bdev);
     395          17 :         free(raid_bdev->base_bdev_info);
     396          17 :         free(raid_bdev->bdev.name);
     397          17 :         free(raid_bdev);
     398          17 : }
     399             : 
     400             : static void
     401           3 : raid_bdev_cleanup_and_free(struct raid_bdev *raid_bdev)
     402             : {
     403           3 :         raid_bdev_cleanup(raid_bdev);
     404           3 :         raid_bdev_free(raid_bdev);
     405           3 : }
     406             : 
     407             : static void
     408         510 : raid_bdev_deconfigure_base_bdev(struct raid_base_bdev_info *base_info)
     409             : {
     410         510 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
     411             : 
     412         510 :         assert(base_info->is_configured);
     413         510 :         assert(raid_bdev->num_base_bdevs_discovered);
     414         510 :         raid_bdev->num_base_bdevs_discovered--;
     415         510 :         base_info->is_configured = false;
     416         510 :         base_info->is_process_target = false;
     417         510 : }
     418             : 
     419             : /*
     420             :  * brief:
     421             :  * free resource of base bdev for raid bdev
     422             :  * params:
     423             :  * base_info - raid base bdev info
     424             :  * returns:
     425             :  * none
     426             :  */
     427             : static void
     428         543 : raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info)
     429             : {
     430         543 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
     431             : 
     432         543 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
     433         543 :         assert(base_info->configure_cb == NULL);
     434             : 
     435         543 :         free(base_info->name);
     436         543 :         base_info->name = NULL;
     437         543 :         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
     438         448 :                 spdk_uuid_set_null(&base_info->uuid);
     439             :         }
     440         543 :         base_info->is_failed = false;
     441             : 
     442         543 :         if (base_info->desc == NULL) {
     443          33 :                 return;
     444             :         }
     445             : 
     446         510 :         spdk_bdev_module_release_bdev(spdk_bdev_desc_get_bdev(base_info->desc));
     447         510 :         spdk_bdev_close(base_info->desc);
     448         510 :         base_info->desc = NULL;
     449         510 :         spdk_put_io_channel(base_info->app_thread_ch);
     450         510 :         base_info->app_thread_ch = NULL;
     451             : 
     452         510 :         if (base_info->is_configured) {
     453         510 :                 raid_bdev_deconfigure_base_bdev(base_info);
     454             :         }
     455             : }
     456             : 
     457             : static void
     458          14 : raid_bdev_io_device_unregister_cb(void *io_device)
     459             : {
     460          14 :         struct raid_bdev *raid_bdev = io_device;
     461             : 
     462          14 :         if (raid_bdev->num_base_bdevs_discovered == 0) {
     463             :                 /* Free raid_bdev when there are no base bdevs left */
     464          14 :                 SPDK_DEBUGLOG(bdev_raid, "raid bdev base bdevs is 0, going to free all in destruct\n");
     465          14 :                 raid_bdev_cleanup(raid_bdev);
     466          14 :                 spdk_bdev_destruct_done(&raid_bdev->bdev, 0);
     467          14 :                 raid_bdev_free(raid_bdev);
     468             :         } else {
     469           0 :                 spdk_bdev_destruct_done(&raid_bdev->bdev, 0);
     470             :         }
     471          14 : }
     472             : 
     473             : void
     474          14 : raid_bdev_module_stop_done(struct raid_bdev *raid_bdev)
     475             : {
     476          14 :         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
     477          14 :                 spdk_io_device_unregister(raid_bdev, raid_bdev_io_device_unregister_cb);
     478             :         }
     479          14 : }
     480             : 
     481             : static void
     482          14 : _raid_bdev_destruct(void *ctxt)
     483             : {
     484          14 :         struct raid_bdev *raid_bdev = ctxt;
     485             :         struct raid_base_bdev_info *base_info;
     486             : 
     487          14 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_destruct\n");
     488             : 
     489          14 :         assert(raid_bdev->process == NULL);
     490             : 
     491         462 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     492             :                 /*
     493             :                  * Close all base bdev descriptors for which call has come from below
     494             :                  * layers.  Also close the descriptors if we have started shutdown.
     495             :                  */
     496         448 :                 if (g_shutdown_started || base_info->remove_scheduled == true) {
     497         448 :                         raid_bdev_free_base_bdev_resource(base_info);
     498             :                 }
     499             :         }
     500             : 
     501          14 :         if (g_shutdown_started) {
     502           0 :                 raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
     503             :         }
     504             : 
     505          14 :         if (raid_bdev->module->stop != NULL) {
     506           0 :                 if (raid_bdev->module->stop(raid_bdev) == false) {
     507           0 :                         return;
     508             :                 }
     509             :         }
     510             : 
     511          14 :         raid_bdev_module_stop_done(raid_bdev);
     512             : }
     513             : 
     514             : static int
     515          14 : raid_bdev_destruct(void *ctx)
     516             : {
     517          14 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_destruct, ctx);
     518             : 
     519          14 :         return 1;
     520             : }
     521             : 
     522             : int
     523           0 : raid_bdev_remap_dix_reftag(void *md_buf, uint64_t num_blocks,
     524             :                            struct spdk_bdev *bdev, uint32_t remapped_offset)
     525             : {
     526           0 :         struct spdk_dif_ctx dif_ctx;
     527           0 :         struct spdk_dif_error err_blk = {};
     528             :         int rc;
     529           0 :         struct spdk_dif_ctx_init_ext_opts dif_opts;
     530           0 :         struct iovec md_iov = {
     531             :                 .iov_base       = md_buf,
     532           0 :                 .iov_len        = num_blocks * bdev->md_len,
     533             :         };
     534             : 
     535           0 :         if (md_buf == NULL) {
     536           0 :                 return 0;
     537             :         }
     538             : 
     539           0 :         dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     540           0 :         dif_opts.dif_pi_format = bdev->dif_pi_format;
     541           0 :         rc = spdk_dif_ctx_init(&dif_ctx,
     542           0 :                                bdev->blocklen, bdev->md_len, bdev->md_interleave,
     543           0 :                                bdev->dif_is_head_of_md, bdev->dif_type,
     544             :                                SPDK_DIF_FLAGS_REFTAG_CHECK,
     545             :                                0, 0, 0, 0, 0, &dif_opts);
     546           0 :         if (rc != 0) {
     547           0 :                 SPDK_ERRLOG("Initialization of DIF context failed\n");
     548           0 :                 return rc;
     549             :         }
     550             : 
     551           0 :         spdk_dif_ctx_set_remapped_init_ref_tag(&dif_ctx, remapped_offset);
     552             : 
     553           0 :         rc = spdk_dix_remap_ref_tag(&md_iov, num_blocks, &dif_ctx, &err_blk, false);
     554           0 :         if (rc != 0) {
     555           0 :                 SPDK_ERRLOG("Remapping reference tag failed. type=%d, offset=%d"
     556             :                             PRIu32 "\n", err_blk.err_type, err_blk.err_offset);
     557             :         }
     558             : 
     559           0 :         return rc;
     560             : }
     561             : 
     562             : int
     563           0 : raid_bdev_verify_dix_reftag(struct iovec *iovs, int iovcnt, void *md_buf,
     564             :                             uint64_t num_blocks, struct spdk_bdev *bdev, uint32_t offset_blocks)
     565             : {
     566           0 :         struct spdk_dif_ctx dif_ctx;
     567           0 :         struct spdk_dif_error err_blk = {};
     568             :         int rc;
     569           0 :         struct spdk_dif_ctx_init_ext_opts dif_opts;
     570           0 :         struct iovec md_iov = {
     571             :                 .iov_base       = md_buf,
     572           0 :                 .iov_len        = num_blocks * bdev->md_len,
     573             :         };
     574             : 
     575           0 :         if (md_buf == NULL) {
     576           0 :                 return 0;
     577             :         }
     578             : 
     579           0 :         dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
     580           0 :         dif_opts.dif_pi_format = bdev->dif_pi_format;
     581           0 :         rc = spdk_dif_ctx_init(&dif_ctx,
     582           0 :                                bdev->blocklen, bdev->md_len, bdev->md_interleave,
     583           0 :                                bdev->dif_is_head_of_md, bdev->dif_type,
     584             :                                SPDK_DIF_FLAGS_REFTAG_CHECK,
     585             :                                offset_blocks, 0, 0, 0, 0, &dif_opts);
     586           0 :         if (rc != 0) {
     587           0 :                 SPDK_ERRLOG("Initialization of DIF context failed\n");
     588           0 :                 return rc;
     589             :         }
     590             : 
     591           0 :         rc = spdk_dix_verify(iovs, iovcnt, &md_iov, num_blocks, &dif_ctx, &err_blk);
     592           0 :         if (rc != 0) {
     593           0 :                 SPDK_ERRLOG("Reference tag check failed. type=%d, offset=%d"
     594             :                             PRIu32 "\n", err_blk.err_type, err_blk.err_offset);
     595             :         }
     596             : 
     597           0 :         return rc;
     598             : }
     599             : 
     600             : void
     601          11 : raid_bdev_io_complete(struct raid_bdev_io *raid_io, enum spdk_bdev_io_status status)
     602             : {
     603          11 :         struct spdk_bdev_io *bdev_io = spdk_bdev_io_from_ctx(raid_io);
     604             :         int rc;
     605             : 
     606          11 :         if (raid_io->split.offset != RAID_OFFSET_BLOCKS_INVALID) {
     607          10 :                 struct iovec *split_iov = raid_io->split.iov;
     608          10 :                 const struct iovec *split_iov_orig = &raid_io->split.iov_copy;
     609             : 
     610             :                 /*
     611             :                  * Non-zero offset here means that this is the completion of the first part of the
     612             :                  * split I/O (the higher LBAs). Then, we submit the second part and set offset to 0.
     613             :                  */
     614          10 :                 if (raid_io->split.offset != 0) {
     615           5 :                         raid_io->offset_blocks = bdev_io->u.bdev.offset_blocks;
     616           5 :                         raid_io->md_buf = bdev_io->u.bdev.md_buf;
     617             : 
     618           5 :                         if (status == SPDK_BDEV_IO_STATUS_SUCCESS) {
     619           5 :                                 raid_io->num_blocks = raid_io->split.offset;
     620           5 :                                 raid_io->iovcnt = raid_io->iovs - bdev_io->u.bdev.iovs;
     621           5 :                                 raid_io->iovs = bdev_io->u.bdev.iovs;
     622           5 :                                 if (split_iov != NULL) {
     623           4 :                                         raid_io->iovcnt++;
     624           4 :                                         split_iov->iov_len = split_iov->iov_base - split_iov_orig->iov_base;
     625           4 :                                         split_iov->iov_base = split_iov_orig->iov_base;
     626             :                                 }
     627             : 
     628           5 :                                 raid_io->split.offset = 0;
     629           5 :                                 raid_io->base_bdev_io_submitted = 0;
     630           5 :                                 raid_io->raid_ch = raid_io->raid_ch->process.ch_processed;
     631             : 
     632           5 :                                 raid_io->raid_bdev->module->submit_rw_request(raid_io);
     633           5 :                                 return;
     634             :                         }
     635             :                 }
     636             : 
     637           5 :                 raid_io->num_blocks = bdev_io->u.bdev.num_blocks;
     638           5 :                 raid_io->iovcnt = bdev_io->u.bdev.iovcnt;
     639           5 :                 raid_io->iovs = bdev_io->u.bdev.iovs;
     640           5 :                 if (split_iov != NULL) {
     641           4 :                         *split_iov = *split_iov_orig;
     642             :                 }
     643             :         }
     644             : 
     645           6 :         if (spdk_unlikely(raid_io->completion_cb != NULL)) {
     646           0 :                 raid_io->completion_cb(raid_io, status);
     647             :         } else {
     648           6 :                 if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
     649             :                                   spdk_bdev_get_dif_type(bdev_io->bdev) != SPDK_DIF_DISABLE &&
     650             :                                   bdev_io->bdev->dif_check_flags & SPDK_DIF_FLAGS_REFTAG_CHECK &&
     651             :                                   status == SPDK_BDEV_IO_STATUS_SUCCESS)) {
     652             : 
     653           0 :                         rc = raid_bdev_remap_dix_reftag(bdev_io->u.bdev.md_buf,
     654             :                                                         bdev_io->u.bdev.num_blocks, bdev_io->bdev,
     655           0 :                                                         bdev_io->u.bdev.offset_blocks);
     656           0 :                         if (rc != 0) {
     657           0 :                                 status = SPDK_BDEV_IO_STATUS_FAILED;
     658             :                         }
     659             :                 }
     660           6 :                 spdk_bdev_io_complete(bdev_io, status);
     661             :         }
     662             : }
     663             : 
     664             : /*
     665             :  * brief:
     666             :  * raid_bdev_io_complete_part - signal the completion of a part of the expected
     667             :  * base bdev IOs and complete the raid_io if this is the final expected IO.
     668             :  * The caller should first set raid_io->base_bdev_io_remaining. This function
     669             :  * will decrement this counter by the value of the 'completed' parameter and
     670             :  * complete the raid_io if the counter reaches 0. The caller is free to
     671             :  * interpret the 'base_bdev_io_remaining' and 'completed' values as needed,
     672             :  * it can represent e.g. blocks or IOs.
     673             :  * params:
     674             :  * raid_io - pointer to raid_bdev_io
     675             :  * completed - the part of the raid_io that has been completed
     676             :  * status - status of the base IO
     677             :  * returns:
     678             :  * true - if the raid_io is completed
     679             :  * false - otherwise
     680             :  */
     681             : bool
     682          32 : raid_bdev_io_complete_part(struct raid_bdev_io *raid_io, uint64_t completed,
     683             :                            enum spdk_bdev_io_status status)
     684             : {
     685          32 :         assert(raid_io->base_bdev_io_remaining >= completed);
     686          32 :         raid_io->base_bdev_io_remaining -= completed;
     687             : 
     688          32 :         if (status != raid_io->base_bdev_io_status_default) {
     689           0 :                 raid_io->base_bdev_io_status = status;
     690             :         }
     691             : 
     692          32 :         if (raid_io->base_bdev_io_remaining == 0) {
     693           1 :                 raid_bdev_io_complete(raid_io, raid_io->base_bdev_io_status);
     694           1 :                 return true;
     695             :         } else {
     696          31 :                 return false;
     697             :         }
     698             : }
     699             : 
     700             : /*
     701             :  * brief:
     702             :  * raid_bdev_queue_io_wait function processes the IO which failed to submit.
     703             :  * It will try to queue the IOs after storing the context to bdev wait queue logic.
     704             :  * params:
     705             :  * raid_io - pointer to raid_bdev_io
     706             :  * bdev - the block device that the IO is submitted to
     707             :  * ch - io channel
     708             :  * cb_fn - callback when the spdk_bdev_io for bdev becomes available
     709             :  * returns:
     710             :  * none
     711             :  */
     712             : void
     713           0 : raid_bdev_queue_io_wait(struct raid_bdev_io *raid_io, struct spdk_bdev *bdev,
     714             :                         struct spdk_io_channel *ch, spdk_bdev_io_wait_cb cb_fn)
     715             : {
     716           0 :         raid_io->waitq_entry.bdev = bdev;
     717           0 :         raid_io->waitq_entry.cb_fn = cb_fn;
     718           0 :         raid_io->waitq_entry.cb_arg = raid_io;
     719           0 :         spdk_bdev_queue_io_wait(bdev, ch, &raid_io->waitq_entry);
     720           0 : }
     721             : 
     722             : static void
     723          32 : raid_base_bdev_reset_complete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
     724             : {
     725          32 :         struct raid_bdev_io *raid_io = cb_arg;
     726             : 
     727          32 :         spdk_bdev_free_io(bdev_io);
     728             : 
     729          32 :         raid_bdev_io_complete_part(raid_io, 1, success ?
     730             :                                    SPDK_BDEV_IO_STATUS_SUCCESS :
     731             :                                    SPDK_BDEV_IO_STATUS_FAILED);
     732          32 : }
     733             : 
     734             : static void raid_bdev_submit_reset_request(struct raid_bdev_io *raid_io);
     735             : 
     736             : static void
     737           0 : _raid_bdev_submit_reset_request(void *_raid_io)
     738             : {
     739           0 :         struct raid_bdev_io *raid_io = _raid_io;
     740             : 
     741           0 :         raid_bdev_submit_reset_request(raid_io);
     742           0 : }
     743             : 
     744             : /*
     745             :  * brief:
     746             :  * raid_bdev_submit_reset_request function submits reset requests
     747             :  * to member disks; it will submit as many as possible unless a reset fails with -ENOMEM, in
     748             :  * which case it will queue it for later submission
     749             :  * params:
     750             :  * raid_io
     751             :  * returns:
     752             :  * none
     753             :  */
     754             : static void
     755           1 : raid_bdev_submit_reset_request(struct raid_bdev_io *raid_io)
     756             : {
     757             :         struct raid_bdev                *raid_bdev;
     758             :         int                             ret;
     759             :         uint8_t                         i;
     760             :         struct raid_base_bdev_info      *base_info;
     761             :         struct spdk_io_channel          *base_ch;
     762             : 
     763           1 :         raid_bdev = raid_io->raid_bdev;
     764             : 
     765           1 :         if (raid_io->base_bdev_io_remaining == 0) {
     766           1 :                 raid_io->base_bdev_io_remaining = raid_bdev->num_base_bdevs;
     767             :         }
     768             : 
     769          33 :         for (i = raid_io->base_bdev_io_submitted; i < raid_bdev->num_base_bdevs; i++) {
     770          32 :                 base_info = &raid_bdev->base_bdev_info[i];
     771          32 :                 base_ch = raid_io->raid_ch->base_channel[i];
     772          32 :                 if (base_ch == NULL) {
     773           0 :                         raid_io->base_bdev_io_submitted++;
     774           0 :                         raid_bdev_io_complete_part(raid_io, 1, SPDK_BDEV_IO_STATUS_SUCCESS);
     775           0 :                         continue;
     776             :                 }
     777          32 :                 ret = spdk_bdev_reset(base_info->desc, base_ch,
     778             :                                       raid_base_bdev_reset_complete, raid_io);
     779          32 :                 if (ret == 0) {
     780          32 :                         raid_io->base_bdev_io_submitted++;
     781           0 :                 } else if (ret == -ENOMEM) {
     782           0 :                         raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
     783             :                                                 base_ch, _raid_bdev_submit_reset_request);
     784           0 :                         return;
     785             :                 } else {
     786           0 :                         SPDK_ERRLOG("bdev io submit error not due to ENOMEM, it should not happen\n");
     787           0 :                         assert(false);
     788             :                         raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     789             :                         return;
     790             :                 }
     791             :         }
     792             : }
     793             : 
     794             : static void
     795           5 : raid_bdev_io_split(struct raid_bdev_io *raid_io, uint64_t split_offset)
     796             : {
     797           5 :         struct raid_bdev *raid_bdev = raid_io->raid_bdev;
     798           5 :         size_t iov_offset = split_offset * raid_bdev->bdev.blocklen;
     799             :         int i;
     800             : 
     801           5 :         assert(split_offset != 0);
     802           5 :         assert(raid_io->split.offset == RAID_OFFSET_BLOCKS_INVALID);
     803           5 :         raid_io->split.offset = split_offset;
     804             : 
     805           5 :         raid_io->offset_blocks += split_offset;
     806           5 :         raid_io->num_blocks -= split_offset;
     807           5 :         if (raid_io->md_buf != NULL) {
     808           5 :                 raid_io->md_buf += (split_offset * raid_bdev->bdev.md_len);
     809             :         }
     810             : 
     811          12 :         for (i = 0; i < raid_io->iovcnt; i++) {
     812          12 :                 struct iovec *iov = &raid_io->iovs[i];
     813             : 
     814          12 :                 if (iov_offset < iov->iov_len) {
     815           5 :                         if (iov_offset == 0) {
     816           1 :                                 raid_io->split.iov = NULL;
     817             :                         } else {
     818           4 :                                 raid_io->split.iov = iov;
     819           4 :                                 raid_io->split.iov_copy = *iov;
     820           4 :                                 iov->iov_base += iov_offset;
     821           4 :                                 iov->iov_len -= iov_offset;
     822             :                         }
     823           5 :                         raid_io->iovs += i;
     824           5 :                         raid_io->iovcnt -= i;
     825           5 :                         break;
     826             :                 }
     827             : 
     828           7 :                 iov_offset -= iov->iov_len;
     829             :         }
     830           5 : }
     831             : 
     832             : static void
     833           5 : raid_bdev_submit_rw_request(struct raid_bdev_io *raid_io)
     834             : {
     835           5 :         struct raid_bdev_io_channel *raid_ch = raid_io->raid_ch;
     836             : 
     837           5 :         if (raid_ch->process.offset != RAID_OFFSET_BLOCKS_INVALID) {
     838           5 :                 uint64_t offset_begin = raid_io->offset_blocks;
     839           5 :                 uint64_t offset_end = offset_begin + raid_io->num_blocks;
     840             : 
     841           5 :                 if (offset_end > raid_ch->process.offset) {
     842           5 :                         if (offset_begin < raid_ch->process.offset) {
     843             :                                 /*
     844             :                                  * If the I/O spans both the processed and unprocessed ranges,
     845             :                                  * split it and first handle the unprocessed part. After it
     846             :                                  * completes, the rest will be handled.
     847             :                                  * This situation occurs when the process thread is not active
     848             :                                  * or is waiting for the process window range to be locked
     849             :                                  * (quiesced). When a window is being processed, such I/Os will be
     850             :                                  * deferred by the bdev layer until the window is unlocked.
     851             :                                  */
     852           5 :                                 SPDK_DEBUGLOG(bdev_raid, "split: process_offset: %lu offset_begin: %lu offset_end: %lu\n",
     853             :                                               raid_ch->process.offset, offset_begin, offset_end);
     854           5 :                                 raid_bdev_io_split(raid_io, raid_ch->process.offset - offset_begin);
     855             :                         }
     856             :                 } else {
     857             :                         /* Use the child channel, which corresponds to the already processed range */
     858           0 :                         raid_io->raid_ch = raid_ch->process.ch_processed;
     859             :                 }
     860             :         }
     861             : 
     862           5 :         raid_io->raid_bdev->module->submit_rw_request(raid_io);
     863           5 : }
     864             : 
     865             : /*
     866             :  * brief:
     867             :  * Callback function to spdk_bdev_io_get_buf.
     868             :  * params:
     869             :  * ch - pointer to raid bdev io channel
     870             :  * bdev_io - pointer to parent bdev_io on raid bdev device
     871             :  * success - True if buffer is allocated or false otherwise.
     872             :  * returns:
     873             :  * none
     874             :  */
     875             : static void
     876           0 : raid_bdev_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
     877             :                      bool success)
     878             : {
     879           0 :         struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx;
     880             : 
     881           0 :         if (!success) {
     882           0 :                 raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     883           0 :                 return;
     884             :         }
     885             : 
     886           0 :         raid_bdev_submit_rw_request(raid_io);
     887             : }
     888             : 
     889             : void
     890           6 : raid_bdev_io_init(struct raid_bdev_io *raid_io, struct raid_bdev_io_channel *raid_ch,
     891             :                   enum spdk_bdev_io_type type, uint64_t offset_blocks,
     892             :                   uint64_t num_blocks, struct iovec *iovs, int iovcnt, void *md_buf,
     893             :                   struct spdk_memory_domain *memory_domain, void *memory_domain_ctx)
     894             : {
     895           6 :         struct spdk_io_channel *ch = spdk_io_channel_from_ctx(raid_ch);
     896           6 :         struct raid_bdev *raid_bdev = spdk_io_channel_get_io_device(ch);
     897             : 
     898           6 :         raid_io->type = type;
     899           6 :         raid_io->offset_blocks = offset_blocks;
     900           6 :         raid_io->num_blocks = num_blocks;
     901           6 :         raid_io->iovs = iovs;
     902           6 :         raid_io->iovcnt = iovcnt;
     903           6 :         raid_io->memory_domain = memory_domain;
     904           6 :         raid_io->memory_domain_ctx = memory_domain_ctx;
     905           6 :         raid_io->md_buf = md_buf;
     906             : 
     907           6 :         raid_io->raid_bdev = raid_bdev;
     908           6 :         raid_io->raid_ch = raid_ch;
     909           6 :         raid_io->base_bdev_io_remaining = 0;
     910           6 :         raid_io->base_bdev_io_submitted = 0;
     911           6 :         raid_io->completion_cb = NULL;
     912           6 :         raid_io->split.offset = RAID_OFFSET_BLOCKS_INVALID;
     913             : 
     914           6 :         raid_bdev_io_set_default_status(raid_io, SPDK_BDEV_IO_STATUS_SUCCESS);
     915           6 : }
     916             : 
     917             : /*
     918             :  * brief:
     919             :  * raid_bdev_submit_request function is the submit_request function pointer of
     920             :  * raid bdev function table. This is used to submit the io on raid_bdev to below
     921             :  * layers.
     922             :  * params:
     923             :  * ch - pointer to raid bdev io channel
     924             :  * bdev_io - pointer to parent bdev_io on raid bdev device
     925             :  * returns:
     926             :  * none
     927             :  */
     928             : static void
     929           6 : raid_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
     930             : {
     931           6 :         struct raid_bdev_io *raid_io = (struct raid_bdev_io *)bdev_io->driver_ctx;
     932             : 
     933           6 :         raid_bdev_io_init(raid_io, spdk_io_channel_get_ctx(ch), bdev_io->type,
     934             :                           bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
     935             :                           bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.md_buf,
     936             :                           bdev_io->u.bdev.memory_domain, bdev_io->u.bdev.memory_domain_ctx);
     937             : 
     938           6 :         switch (bdev_io->type) {
     939           0 :         case SPDK_BDEV_IO_TYPE_READ:
     940           0 :                 spdk_bdev_io_get_buf(bdev_io, raid_bdev_get_buf_cb,
     941           0 :                                      bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
     942           0 :                 break;
     943           5 :         case SPDK_BDEV_IO_TYPE_WRITE:
     944           5 :                 raid_bdev_submit_rw_request(raid_io);
     945           5 :                 break;
     946             : 
     947           1 :         case SPDK_BDEV_IO_TYPE_RESET:
     948           1 :                 raid_bdev_submit_reset_request(raid_io);
     949           1 :                 break;
     950             : 
     951           0 :         case SPDK_BDEV_IO_TYPE_FLUSH:
     952             :         case SPDK_BDEV_IO_TYPE_UNMAP:
     953           0 :                 if (raid_io->raid_bdev->process != NULL) {
     954             :                         /* TODO: rebuild support */
     955           0 :                         raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     956           0 :                         return;
     957             :                 }
     958           0 :                 raid_io->raid_bdev->module->submit_null_payload_request(raid_io);
     959           0 :                 break;
     960             : 
     961           0 :         default:
     962           0 :                 SPDK_ERRLOG("submit request, invalid io type %u\n", bdev_io->type);
     963           0 :                 raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
     964           0 :                 break;
     965             :         }
     966             : }
     967             : 
     968             : /*
     969             :  * brief:
     970             :  * _raid_bdev_io_type_supported checks whether io_type is supported in
     971             :  * all base bdev modules of raid bdev module. If anyone among the base_bdevs
     972             :  * doesn't support, the raid device doesn't supports.
     973             :  *
     974             :  * params:
     975             :  * raid_bdev - pointer to raid bdev context
     976             :  * io_type - io type
     977             :  * returns:
     978             :  * true - io_type is supported
     979             :  * false - io_type is not supported
     980             :  */
     981             : inline static bool
     982           1 : _raid_bdev_io_type_supported(struct raid_bdev *raid_bdev, enum spdk_bdev_io_type io_type)
     983             : {
     984             :         struct raid_base_bdev_info *base_info;
     985             : 
     986           1 :         if (io_type == SPDK_BDEV_IO_TYPE_FLUSH ||
     987             :             io_type == SPDK_BDEV_IO_TYPE_UNMAP) {
     988           0 :                 if (raid_bdev->module->submit_null_payload_request == NULL) {
     989           0 :                         return false;
     990             :                 }
     991             :         }
     992             : 
     993          33 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
     994          32 :                 if (base_info->desc == NULL) {
     995           0 :                         continue;
     996             :                 }
     997             : 
     998          32 :                 if (spdk_bdev_io_type_supported(spdk_bdev_desc_get_bdev(base_info->desc), io_type) == false) {
     999           0 :                         return false;
    1000             :                 }
    1001             :         }
    1002             : 
    1003           1 :         return true;
    1004             : }
    1005             : 
    1006             : /*
    1007             :  * brief:
    1008             :  * raid_bdev_io_type_supported is the io_supported function for bdev function
    1009             :  * table which returns whether the particular io type is supported or not by
    1010             :  * raid bdev module
    1011             :  * params:
    1012             :  * ctx - pointer to raid bdev context
    1013             :  * type - io type
    1014             :  * returns:
    1015             :  * true - io_type is supported
    1016             :  * false - io_type is not supported
    1017             :  */
    1018             : static bool
    1019           4 : raid_bdev_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type)
    1020             : {
    1021           4 :         switch (io_type) {
    1022           2 :         case SPDK_BDEV_IO_TYPE_READ:
    1023             :         case SPDK_BDEV_IO_TYPE_WRITE:
    1024           2 :                 return true;
    1025             : 
    1026           1 :         case SPDK_BDEV_IO_TYPE_FLUSH:
    1027             :         case SPDK_BDEV_IO_TYPE_RESET:
    1028             :         case SPDK_BDEV_IO_TYPE_UNMAP:
    1029           1 :                 return _raid_bdev_io_type_supported(ctx, io_type);
    1030             : 
    1031           1 :         default:
    1032           1 :                 return false;
    1033             :         }
    1034             : 
    1035             :         return false;
    1036             : }
    1037             : 
    1038             : /*
    1039             :  * brief:
    1040             :  * raid_bdev_get_io_channel is the get_io_channel function table pointer for
    1041             :  * raid bdev. This is used to return the io channel for this raid bdev
    1042             :  * params:
    1043             :  * ctxt - pointer to raid_bdev
    1044             :  * returns:
    1045             :  * pointer to io channel for raid bdev
    1046             :  */
    1047             : static struct spdk_io_channel *
    1048           0 : raid_bdev_get_io_channel(void *ctxt)
    1049             : {
    1050           0 :         struct raid_bdev *raid_bdev = ctxt;
    1051             : 
    1052           0 :         return spdk_get_io_channel(raid_bdev);
    1053             : }
    1054             : 
    1055             : void
    1056           7 : raid_bdev_write_info_json(struct raid_bdev *raid_bdev, struct spdk_json_write_ctx *w)
    1057             : {
    1058             :         struct raid_base_bdev_info *base_info;
    1059             : 
    1060           7 :         assert(raid_bdev != NULL);
    1061           7 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    1062             : 
    1063           7 :         spdk_json_write_named_uuid(w, "uuid", &raid_bdev->bdev.uuid);
    1064           7 :         spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
    1065           7 :         spdk_json_write_named_string(w, "state", raid_bdev_state_to_str(raid_bdev->state));
    1066           7 :         spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
    1067           7 :         spdk_json_write_named_bool(w, "superblock", raid_bdev->superblock_enabled);
    1068           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs", raid_bdev->num_base_bdevs);
    1069           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs_discovered", raid_bdev->num_base_bdevs_discovered);
    1070           7 :         spdk_json_write_named_uint32(w, "num_base_bdevs_operational",
    1071           7 :                                      raid_bdev->num_base_bdevs_operational);
    1072           7 :         if (raid_bdev->process) {
    1073           0 :                 struct raid_bdev_process *process = raid_bdev->process;
    1074           0 :                 uint64_t offset = process->window_offset;
    1075             : 
    1076           0 :                 spdk_json_write_named_object_begin(w, "process");
    1077           0 :                 spdk_json_write_name(w, "type");
    1078           0 :                 spdk_json_write_string(w, raid_bdev_process_to_str(process->type));
    1079           0 :                 spdk_json_write_named_string(w, "target", process->target->name);
    1080           0 :                 spdk_json_write_named_object_begin(w, "progress");
    1081           0 :                 spdk_json_write_named_uint64(w, "blocks", offset);
    1082           0 :                 spdk_json_write_named_uint32(w, "percent", offset * 100.0 / raid_bdev->bdev.blockcnt);
    1083           0 :                 spdk_json_write_object_end(w);
    1084           0 :                 spdk_json_write_object_end(w);
    1085             :         }
    1086           7 :         spdk_json_write_name(w, "base_bdevs_list");
    1087           7 :         spdk_json_write_array_begin(w);
    1088         231 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1089         224 :                 spdk_json_write_object_begin(w);
    1090         224 :                 spdk_json_write_name(w, "name");
    1091         224 :                 if (base_info->name) {
    1092         224 :                         spdk_json_write_string(w, base_info->name);
    1093             :                 } else {
    1094           0 :                         spdk_json_write_null(w);
    1095             :                 }
    1096         224 :                 spdk_json_write_named_uuid(w, "uuid", &base_info->uuid);
    1097         224 :                 spdk_json_write_named_bool(w, "is_configured", base_info->is_configured);
    1098         224 :                 spdk_json_write_named_uint64(w, "data_offset", base_info->data_offset);
    1099         224 :                 spdk_json_write_named_uint64(w, "data_size", base_info->data_size);
    1100         224 :                 spdk_json_write_object_end(w);
    1101             :         }
    1102           7 :         spdk_json_write_array_end(w);
    1103           7 : }
    1104             : 
    1105             : /*
    1106             :  * brief:
    1107             :  * raid_bdev_dump_info_json is the function table pointer for raid bdev
    1108             :  * params:
    1109             :  * ctx - pointer to raid_bdev
    1110             :  * w - pointer to json context
    1111             :  * returns:
    1112             :  * 0 - success
    1113             :  * non zero - failure
    1114             :  */
    1115             : static int
    1116           1 : raid_bdev_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
    1117             : {
    1118           1 :         struct raid_bdev *raid_bdev = ctx;
    1119             : 
    1120           1 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_dump_config_json\n");
    1121             : 
    1122             :         /* Dump the raid bdev configuration related information */
    1123           1 :         spdk_json_write_named_object_begin(w, "raid");
    1124           1 :         raid_bdev_write_info_json(raid_bdev, w);
    1125           1 :         spdk_json_write_object_end(w);
    1126             : 
    1127           1 :         return 0;
    1128             : }
    1129             : 
    1130             : /*
    1131             :  * brief:
    1132             :  * raid_bdev_write_config_json is the function table pointer for raid bdev
    1133             :  * params:
    1134             :  * bdev - pointer to spdk_bdev
    1135             :  * w - pointer to json context
    1136             :  * returns:
    1137             :  * none
    1138             :  */
    1139             : static void
    1140           0 : raid_bdev_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
    1141             : {
    1142           0 :         struct raid_bdev *raid_bdev = bdev->ctxt;
    1143             :         struct raid_base_bdev_info *base_info;
    1144             : 
    1145           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    1146             : 
    1147           0 :         if (raid_bdev->superblock_enabled) {
    1148             :                 /* raid bdev configuration is stored in the superblock */
    1149           0 :                 return;
    1150             :         }
    1151             : 
    1152           0 :         spdk_json_write_object_begin(w);
    1153             : 
    1154           0 :         spdk_json_write_named_string(w, "method", "bdev_raid_create");
    1155             : 
    1156           0 :         spdk_json_write_named_object_begin(w, "params");
    1157           0 :         spdk_json_write_named_string(w, "name", bdev->name);
    1158           0 :         spdk_json_write_named_uuid(w, "uuid", &raid_bdev->bdev.uuid);
    1159           0 :         if (raid_bdev->strip_size_kb != 0) {
    1160           0 :                 spdk_json_write_named_uint32(w, "strip_size_kb", raid_bdev->strip_size_kb);
    1161             :         }
    1162           0 :         spdk_json_write_named_string(w, "raid_level", raid_bdev_level_to_str(raid_bdev->level));
    1163             : 
    1164           0 :         spdk_json_write_named_array_begin(w, "base_bdevs");
    1165           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1166           0 :                 if (base_info->name) {
    1167           0 :                         spdk_json_write_string(w, base_info->name);
    1168             :                 } else {
    1169           0 :                         char str[32];
    1170             : 
    1171           0 :                         snprintf(str, sizeof(str), "removed_base_bdev_%u", raid_bdev_base_bdev_slot(base_info));
    1172           0 :                         spdk_json_write_string(w, str);
    1173             :                 }
    1174             :         }
    1175           0 :         spdk_json_write_array_end(w);
    1176           0 :         spdk_json_write_object_end(w);
    1177             : 
    1178           0 :         spdk_json_write_object_end(w);
    1179             : }
    1180             : 
    1181             : static int
    1182           0 : raid_bdev_get_memory_domains(void *ctx, struct spdk_memory_domain **domains, int array_size)
    1183             : {
    1184           0 :         struct raid_bdev *raid_bdev = ctx;
    1185             :         struct raid_base_bdev_info *base_info;
    1186           0 :         int domains_count = 0, rc = 0;
    1187             : 
    1188           0 :         if (raid_bdev->module->memory_domains_supported == false) {
    1189           0 :                 return 0;
    1190             :         }
    1191             : 
    1192             :         /* First loop to get the number of memory domains */
    1193           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1194           0 :                 if (base_info->is_configured == false) {
    1195           0 :                         continue;
    1196             :                 }
    1197           0 :                 rc = spdk_bdev_get_memory_domains(spdk_bdev_desc_get_bdev(base_info->desc), NULL, 0);
    1198           0 :                 if (rc < 0) {
    1199           0 :                         return rc;
    1200             :                 }
    1201           0 :                 domains_count += rc;
    1202             :         }
    1203             : 
    1204           0 :         if (!domains || array_size < domains_count) {
    1205           0 :                 return domains_count;
    1206             :         }
    1207             : 
    1208           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1209           0 :                 if (base_info->is_configured == false) {
    1210           0 :                         continue;
    1211             :                 }
    1212           0 :                 rc = spdk_bdev_get_memory_domains(spdk_bdev_desc_get_bdev(base_info->desc), domains, array_size);
    1213           0 :                 if (rc < 0) {
    1214           0 :                         return rc;
    1215             :                 }
    1216           0 :                 domains += rc;
    1217           0 :                 array_size -= rc;
    1218             :         }
    1219             : 
    1220           0 :         return domains_count;
    1221             : }
    1222             : 
    1223             : /* g_raid_bdev_fn_table is the function table for raid bdev */
    1224             : static const struct spdk_bdev_fn_table g_raid_bdev_fn_table = {
    1225             :         .destruct               = raid_bdev_destruct,
    1226             :         .submit_request         = raid_bdev_submit_request,
    1227             :         .io_type_supported      = raid_bdev_io_type_supported,
    1228             :         .get_io_channel         = raid_bdev_get_io_channel,
    1229             :         .dump_info_json         = raid_bdev_dump_info_json,
    1230             :         .write_config_json      = raid_bdev_write_config_json,
    1231             :         .get_memory_domains     = raid_bdev_get_memory_domains,
    1232             : };
    1233             : 
    1234             : struct raid_bdev *
    1235          36 : raid_bdev_find_by_name(const char *name)
    1236             : {
    1237             :         struct raid_bdev *raid_bdev;
    1238             : 
    1239          43 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1240          23 :                 if (strcmp(raid_bdev->bdev.name, name) == 0) {
    1241          16 :                         return raid_bdev;
    1242             :                 }
    1243             :         }
    1244             : 
    1245          20 :         return NULL;
    1246             : }
    1247             : 
    1248             : static struct raid_bdev *
    1249           0 : raid_bdev_find_by_uuid(const struct spdk_uuid *uuid)
    1250             : {
    1251             :         struct raid_bdev *raid_bdev;
    1252             : 
    1253           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1254           0 :                 if (spdk_uuid_compare(&raid_bdev->bdev.uuid, uuid) == 0) {
    1255           0 :                         return raid_bdev;
    1256             :                 }
    1257             :         }
    1258             : 
    1259           0 :         return NULL;
    1260             : }
    1261             : 
    1262             : static struct {
    1263             :         const char *name;
    1264             :         enum raid_level value;
    1265             : } g_raid_level_names[] = {
    1266             :         { "raid0", RAID0 },
    1267             :         { "0", RAID0 },
    1268             :         { "raid1", RAID1 },
    1269             :         { "1", RAID1 },
    1270             :         { "raid5f", RAID5F },
    1271             :         { "5f", RAID5F },
    1272             :         { "concat", CONCAT },
    1273             :         { }
    1274             : };
    1275             : 
    1276             : const char *g_raid_state_names[] = {
    1277             :         [RAID_BDEV_STATE_ONLINE]        = "online",
    1278             :         [RAID_BDEV_STATE_CONFIGURING]   = "configuring",
    1279             :         [RAID_BDEV_STATE_OFFLINE]       = "offline",
    1280             :         [RAID_BDEV_STATE_MAX]           = NULL
    1281             : };
    1282             : 
    1283             : static const char *g_raid_process_type_names[] = {
    1284             :         [RAID_PROCESS_NONE]     = "none",
    1285             :         [RAID_PROCESS_REBUILD]  = "rebuild",
    1286             :         [RAID_PROCESS_MAX]      = NULL
    1287             : };
    1288             : 
    1289             : /* We have to use the typedef in the function declaration to appease astyle. */
    1290             : typedef enum raid_level raid_level_t;
    1291             : typedef enum raid_bdev_state raid_bdev_state_t;
    1292             : 
    1293             : raid_level_t
    1294           4 : raid_bdev_str_to_level(const char *str)
    1295             : {
    1296             :         unsigned int i;
    1297             : 
    1298           4 :         assert(str != NULL);
    1299             : 
    1300          12 :         for (i = 0; g_raid_level_names[i].name != NULL; i++) {
    1301          11 :                 if (strcasecmp(g_raid_level_names[i].name, str) == 0) {
    1302           3 :                         return g_raid_level_names[i].value;
    1303             :                 }
    1304             :         }
    1305             : 
    1306           1 :         return INVALID_RAID_LEVEL;
    1307             : }
    1308             : 
    1309             : const char *
    1310          11 : raid_bdev_level_to_str(enum raid_level level)
    1311             : {
    1312             :         unsigned int i;
    1313             : 
    1314          81 :         for (i = 0; g_raid_level_names[i].name != NULL; i++) {
    1315          71 :                 if (g_raid_level_names[i].value == level) {
    1316           1 :                         return g_raid_level_names[i].name;
    1317             :                 }
    1318             :         }
    1319             : 
    1320          10 :         return "";
    1321             : }
    1322             : 
    1323             : raid_bdev_state_t
    1324           6 : raid_bdev_str_to_state(const char *str)
    1325             : {
    1326             :         unsigned int i;
    1327             : 
    1328           6 :         assert(str != NULL);
    1329             : 
    1330          18 :         for (i = 0; i < RAID_BDEV_STATE_MAX; i++) {
    1331          15 :                 if (strcasecmp(g_raid_state_names[i], str) == 0) {
    1332           3 :                         break;
    1333             :                 }
    1334             :         }
    1335             : 
    1336           6 :         return i;
    1337             : }
    1338             : 
    1339             : const char *
    1340           7 : raid_bdev_state_to_str(enum raid_bdev_state state)
    1341             : {
    1342           7 :         if (state >= RAID_BDEV_STATE_MAX) {
    1343           0 :                 return "";
    1344             :         }
    1345             : 
    1346           7 :         return g_raid_state_names[state];
    1347             : }
    1348             : 
    1349             : const char *
    1350           6 : raid_bdev_process_to_str(enum raid_process_type value)
    1351             : {
    1352           6 :         if (value >= RAID_PROCESS_MAX) {
    1353           0 :                 return "";
    1354             :         }
    1355             : 
    1356           6 :         return g_raid_process_type_names[value];
    1357             : }
    1358             : 
    1359             : /*
    1360             :  * brief:
    1361             :  * raid_bdev_fini_start is called when bdev layer is starting the
    1362             :  * shutdown process
    1363             :  * params:
    1364             :  * none
    1365             :  * returns:
    1366             :  * none
    1367             :  */
    1368             : static void
    1369           0 : raid_bdev_fini_start(void)
    1370             : {
    1371             :         struct raid_bdev *raid_bdev;
    1372             :         struct raid_base_bdev_info *base_info;
    1373             : 
    1374           0 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_fini_start\n");
    1375             : 
    1376           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1377           0 :                 if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    1378           0 :                         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1379           0 :                                 raid_bdev_free_base_bdev_resource(base_info);
    1380             :                         }
    1381             :                 }
    1382             :         }
    1383             : 
    1384           0 :         g_shutdown_started = true;
    1385           0 : }
    1386             : 
    1387             : /*
    1388             :  * brief:
    1389             :  * raid_bdev_exit is called on raid bdev module exit time by bdev layer
    1390             :  * params:
    1391             :  * none
    1392             :  * returns:
    1393             :  * none
    1394             :  */
    1395             : static void
    1396          12 : raid_bdev_exit(void)
    1397             : {
    1398             :         struct raid_bdev *raid_bdev, *tmp;
    1399             : 
    1400          12 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_exit\n");
    1401             : 
    1402          12 :         TAILQ_FOREACH_SAFE(raid_bdev, &g_raid_bdev_list, global_link, tmp) {
    1403           0 :                 raid_bdev_cleanup_and_free(raid_bdev);
    1404             :         }
    1405          12 : }
    1406             : 
    1407             : static void
    1408           0 : raid_bdev_opts_config_json(struct spdk_json_write_ctx *w)
    1409             : {
    1410           0 :         spdk_json_write_object_begin(w);
    1411             : 
    1412           0 :         spdk_json_write_named_string(w, "method", "bdev_raid_set_options");
    1413             : 
    1414           0 :         spdk_json_write_named_object_begin(w, "params");
    1415           0 :         spdk_json_write_named_uint32(w, "process_window_size_kb", g_opts.process_window_size_kb);
    1416           0 :         spdk_json_write_named_uint32(w, "process_max_bandwidth_mb_sec",
    1417             :                                      g_opts.process_max_bandwidth_mb_sec);
    1418           0 :         spdk_json_write_object_end(w);
    1419             : 
    1420           0 :         spdk_json_write_object_end(w);
    1421           0 : }
    1422             : 
    1423             : static int
    1424           0 : raid_bdev_config_json(struct spdk_json_write_ctx *w)
    1425             : {
    1426           0 :         raid_bdev_opts_config_json(w);
    1427             : 
    1428           0 :         return 0;
    1429             : }
    1430             : 
    1431             : /*
    1432             :  * brief:
    1433             :  * raid_bdev_get_ctx_size is used to return the context size of bdev_io for raid
    1434             :  * module
    1435             :  * params:
    1436             :  * none
    1437             :  * returns:
    1438             :  * size of spdk_bdev_io context for raid
    1439             :  */
    1440             : static int
    1441           1 : raid_bdev_get_ctx_size(void)
    1442             : {
    1443           1 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_get_ctx_size\n");
    1444           1 :         return sizeof(struct raid_bdev_io);
    1445             : }
    1446             : 
    1447             : static struct spdk_bdev_module g_raid_if = {
    1448             :         .name = "raid",
    1449             :         .module_init = raid_bdev_init,
    1450             :         .fini_start = raid_bdev_fini_start,
    1451             :         .module_fini = raid_bdev_exit,
    1452             :         .config_json = raid_bdev_config_json,
    1453             :         .get_ctx_size = raid_bdev_get_ctx_size,
    1454             :         .examine_disk = raid_bdev_examine,
    1455             :         .async_init = false,
    1456             :         .async_fini = false,
    1457             : };
    1458           1 : SPDK_BDEV_MODULE_REGISTER(raid, &g_raid_if)
    1459             : 
    1460             : /*
    1461             :  * brief:
    1462             :  * raid_bdev_init is the initialization function for raid bdev module
    1463             :  * params:
    1464             :  * none
    1465             :  * returns:
    1466             :  * 0 - success
    1467             :  * non zero - failure
    1468             :  */
    1469             : static int
    1470          12 : raid_bdev_init(void)
    1471             : {
    1472          12 :         return 0;
    1473             : }
    1474             : 
    1475             : static int
    1476          20 : _raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
    1477             :                   enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
    1478             :                   struct raid_bdev **raid_bdev_out)
    1479             : {
    1480             :         struct raid_bdev *raid_bdev;
    1481             :         struct spdk_bdev *raid_bdev_gen;
    1482             :         struct raid_bdev_module *module;
    1483             :         struct raid_base_bdev_info *base_info;
    1484             :         uint8_t min_operational;
    1485             : 
    1486          20 :         if (strnlen(name, RAID_BDEV_SB_NAME_SIZE) == RAID_BDEV_SB_NAME_SIZE) {
    1487           0 :                 SPDK_ERRLOG("Raid bdev name '%s' exceeds %d characters\n", name, RAID_BDEV_SB_NAME_SIZE - 1);
    1488           0 :                 return -EINVAL;
    1489             :         }
    1490             : 
    1491          20 :         if (raid_bdev_find_by_name(name) != NULL) {
    1492           1 :                 SPDK_ERRLOG("Duplicate raid bdev name found: %s\n", name);
    1493           1 :                 return -EEXIST;
    1494             :         }
    1495             : 
    1496          19 :         if (level == RAID1) {
    1497           0 :                 if (strip_size != 0) {
    1498           0 :                         SPDK_ERRLOG("Strip size is not supported by raid1\n");
    1499           0 :                         return -EINVAL;
    1500             :                 }
    1501          19 :         } else if (spdk_u32_is_pow2(strip_size) == false) {
    1502           1 :                 SPDK_ERRLOG("Invalid strip size %" PRIu32 "\n", strip_size);
    1503           1 :                 return -EINVAL;
    1504             :         }
    1505             : 
    1506          18 :         module = raid_bdev_module_find(level);
    1507          18 :         if (module == NULL) {
    1508           1 :                 SPDK_ERRLOG("Unsupported raid level '%d'\n", level);
    1509           1 :                 return -EINVAL;
    1510             :         }
    1511             : 
    1512          17 :         assert(module->base_bdevs_min != 0);
    1513          17 :         if (num_base_bdevs < module->base_bdevs_min) {
    1514           0 :                 SPDK_ERRLOG("At least %u base devices required for %s\n",
    1515             :                             module->base_bdevs_min,
    1516             :                             raid_bdev_level_to_str(level));
    1517           0 :                 return -EINVAL;
    1518             :         }
    1519             : 
    1520          17 :         switch (module->base_bdevs_constraint.type) {
    1521           0 :         case CONSTRAINT_MAX_BASE_BDEVS_REMOVED:
    1522           0 :                 min_operational = num_base_bdevs - module->base_bdevs_constraint.value;
    1523           0 :                 break;
    1524           0 :         case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL:
    1525           0 :                 min_operational = module->base_bdevs_constraint.value;
    1526           0 :                 break;
    1527          17 :         case CONSTRAINT_UNSET:
    1528          17 :                 if (module->base_bdevs_constraint.value != 0) {
    1529           0 :                         SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid bdev '%s'.\n",
    1530             :                                     (uint8_t)module->base_bdevs_constraint.value, name);
    1531           0 :                         return -EINVAL;
    1532             :                 }
    1533          17 :                 min_operational = num_base_bdevs;
    1534          17 :                 break;
    1535           0 :         default:
    1536           0 :                 SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid level '%s'.\n",
    1537             :                             (uint8_t)module->base_bdevs_constraint.type,
    1538             :                             raid_bdev_level_to_str(module->level));
    1539           0 :                 return -EINVAL;
    1540             :         };
    1541             : 
    1542          17 :         if (min_operational == 0 || min_operational > num_base_bdevs) {
    1543           0 :                 SPDK_ERRLOG("Wrong constraint value for raid level '%s'.\n",
    1544             :                             raid_bdev_level_to_str(module->level));
    1545           0 :                 return -EINVAL;
    1546             :         }
    1547             : 
    1548          17 :         raid_bdev = calloc(1, sizeof(*raid_bdev));
    1549          17 :         if (!raid_bdev) {
    1550           0 :                 SPDK_ERRLOG("Unable to allocate memory for raid bdev\n");
    1551           0 :                 return -ENOMEM;
    1552             :         }
    1553             : 
    1554          17 :         raid_bdev->module = module;
    1555          17 :         raid_bdev->num_base_bdevs = num_base_bdevs;
    1556          17 :         raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs,
    1557             :                                            sizeof(struct raid_base_bdev_info));
    1558          17 :         if (!raid_bdev->base_bdev_info) {
    1559           0 :                 SPDK_ERRLOG("Unable able to allocate base bdev info\n");
    1560           0 :                 raid_bdev_free(raid_bdev);
    1561           0 :                 return -ENOMEM;
    1562             :         }
    1563             : 
    1564         561 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1565         544 :                 base_info->raid_bdev = raid_bdev;
    1566             :         }
    1567             : 
    1568             :         /* strip_size_kb is from the rpc param.  strip_size is in blocks and used
    1569             :          * internally and set later.
    1570             :          */
    1571          17 :         raid_bdev->strip_size = 0;
    1572          17 :         raid_bdev->strip_size_kb = strip_size;
    1573          17 :         raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
    1574          17 :         raid_bdev->level = level;
    1575          17 :         raid_bdev->min_base_bdevs_operational = min_operational;
    1576          17 :         raid_bdev->superblock_enabled = superblock_enabled;
    1577             : 
    1578          17 :         raid_bdev_gen = &raid_bdev->bdev;
    1579             : 
    1580          17 :         raid_bdev_gen->name = strdup(name);
    1581          17 :         if (!raid_bdev_gen->name) {
    1582           0 :                 SPDK_ERRLOG("Unable to allocate name for raid\n");
    1583           0 :                 raid_bdev_free(raid_bdev);
    1584           0 :                 return -ENOMEM;
    1585             :         }
    1586             : 
    1587          17 :         raid_bdev_gen->product_name = "Raid Volume";
    1588          17 :         raid_bdev_gen->ctxt = raid_bdev;
    1589          17 :         raid_bdev_gen->fn_table = &g_raid_bdev_fn_table;
    1590          17 :         raid_bdev_gen->module = &g_raid_if;
    1591          17 :         raid_bdev_gen->write_cache = 0;
    1592          17 :         spdk_uuid_copy(&raid_bdev_gen->uuid, uuid);
    1593             : 
    1594          17 :         TAILQ_INSERT_TAIL(&g_raid_bdev_list, raid_bdev, global_link);
    1595             : 
    1596          17 :         *raid_bdev_out = raid_bdev;
    1597             : 
    1598          17 :         return 0;
    1599             : }
    1600             : 
    1601             : /*
    1602             :  * brief:
    1603             :  * raid_bdev_create allocates raid bdev based on passed configuration
    1604             :  * params:
    1605             :  * name - name for raid bdev
    1606             :  * strip_size - strip size in KB
    1607             :  * num_base_bdevs - number of base bdevs
    1608             :  * level - raid level
    1609             :  * superblock_enabled - true if raid should have superblock
    1610             :  * uuid - uuid to set for the bdev
    1611             :  * raid_bdev_out - the created raid bdev
    1612             :  * returns:
    1613             :  * 0 - success
    1614             :  * non zero - failure
    1615             :  */
    1616             : int
    1617          20 : raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
    1618             :                  enum raid_level level, bool superblock_enabled, const struct spdk_uuid *uuid,
    1619             :                  struct raid_bdev **raid_bdev_out)
    1620             : {
    1621          20 :         struct raid_bdev *raid_bdev;
    1622             :         int rc;
    1623             : 
    1624          20 :         assert(uuid != NULL);
    1625             : 
    1626          20 :         rc = _raid_bdev_create(name, strip_size, num_base_bdevs, level, superblock_enabled, uuid,
    1627             :                                &raid_bdev);
    1628          20 :         if (rc != 0) {
    1629           3 :                 return rc;
    1630             :         }
    1631             : 
    1632          17 :         if (superblock_enabled && spdk_uuid_is_null(uuid)) {
    1633             :                 /* we need to have the uuid to store in the superblock before the bdev is registered */
    1634           1 :                 spdk_uuid_generate(&raid_bdev->bdev.uuid);
    1635             :         }
    1636             : 
    1637          17 :         raid_bdev->num_base_bdevs_operational = num_base_bdevs;
    1638             : 
    1639          17 :         *raid_bdev_out = raid_bdev;
    1640             : 
    1641          17 :         return 0;
    1642             : }
    1643             : 
    1644             : static void
    1645           0 : _raid_bdev_unregistering_cont(void *ctx)
    1646             : {
    1647           0 :         struct raid_bdev *raid_bdev = ctx;
    1648             : 
    1649           0 :         spdk_bdev_close(raid_bdev->self_desc);
    1650           0 :         raid_bdev->self_desc = NULL;
    1651           0 : }
    1652             : 
    1653             : static void
    1654           0 : raid_bdev_unregistering_cont(void *ctx)
    1655             : {
    1656           0 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_unregistering_cont, ctx);
    1657           0 : }
    1658             : 
    1659             : static int
    1660           0 : raid_bdev_process_add_finish_action(struct raid_bdev_process *process, spdk_msg_fn cb, void *cb_ctx)
    1661             : {
    1662             :         struct raid_process_finish_action *finish_action;
    1663             : 
    1664           0 :         assert(spdk_get_thread() == process->thread);
    1665           0 :         assert(process->state < RAID_PROCESS_STATE_STOPPED);
    1666             : 
    1667           0 :         finish_action = calloc(1, sizeof(*finish_action));
    1668           0 :         if (finish_action == NULL) {
    1669           0 :                 return -ENOMEM;
    1670             :         }
    1671             : 
    1672           0 :         finish_action->cb = cb;
    1673           0 :         finish_action->cb_ctx = cb_ctx;
    1674             : 
    1675           0 :         TAILQ_INSERT_TAIL(&process->finish_actions, finish_action, link);
    1676             : 
    1677           0 :         return 0;
    1678             : }
    1679             : 
    1680             : static void
    1681           0 : raid_bdev_unregistering_stop_process(void *ctx)
    1682             : {
    1683           0 :         struct raid_bdev_process *process = ctx;
    1684           0 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    1685             :         int rc;
    1686             : 
    1687           0 :         process->state = RAID_PROCESS_STATE_STOPPING;
    1688           0 :         if (process->status == 0) {
    1689           0 :                 process->status = -ECANCELED;
    1690             :         }
    1691             : 
    1692           0 :         rc = raid_bdev_process_add_finish_action(process, raid_bdev_unregistering_cont, raid_bdev);
    1693           0 :         if (rc != 0) {
    1694           0 :                 SPDK_ERRLOG("Failed to add raid bdev '%s' process finish action: %s\n",
    1695             :                             raid_bdev->bdev.name, spdk_strerror(-rc));
    1696             :         }
    1697           0 : }
    1698             : 
    1699             : static void
    1700           0 : raid_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx)
    1701             : {
    1702           0 :         struct raid_bdev *raid_bdev = event_ctx;
    1703             : 
    1704           0 :         if (type == SPDK_BDEV_EVENT_REMOVE) {
    1705           0 :                 if (raid_bdev->process != NULL) {
    1706           0 :                         spdk_thread_send_msg(raid_bdev->process->thread, raid_bdev_unregistering_stop_process,
    1707           0 :                                              raid_bdev->process);
    1708             :                 } else {
    1709           0 :                         raid_bdev_unregistering_cont(raid_bdev);
    1710             :                 }
    1711             :         }
    1712           0 : }
    1713             : 
    1714             : static void
    1715          14 : raid_bdev_configure_cont(struct raid_bdev *raid_bdev)
    1716             : {
    1717          14 :         struct spdk_bdev *raid_bdev_gen = &raid_bdev->bdev;
    1718             :         int rc;
    1719             : 
    1720          14 :         raid_bdev->state = RAID_BDEV_STATE_ONLINE;
    1721          14 :         SPDK_DEBUGLOG(bdev_raid, "io device register %p\n", raid_bdev);
    1722          14 :         SPDK_DEBUGLOG(bdev_raid, "blockcnt %" PRIu64 ", blocklen %u\n",
    1723             :                       raid_bdev_gen->blockcnt, raid_bdev_gen->blocklen);
    1724          14 :         spdk_io_device_register(raid_bdev, raid_bdev_create_cb, raid_bdev_destroy_cb,
    1725             :                                 sizeof(struct raid_bdev_io_channel),
    1726          14 :                                 raid_bdev_gen->name);
    1727          14 :         rc = spdk_bdev_register(raid_bdev_gen);
    1728          14 :         if (rc != 0) {
    1729           0 :                 SPDK_ERRLOG("Failed to register raid bdev '%s': %s\n",
    1730             :                             raid_bdev_gen->name, spdk_strerror(-rc));
    1731           0 :                 goto out;
    1732             :         }
    1733             : 
    1734             :         /*
    1735             :          * Open the bdev internally to delay unregistering if we need to stop a background process
    1736             :          * first. The process may still need to unquiesce a range but it will fail because the
    1737             :          * bdev's internal.spinlock is destroyed by the time the destruct callback is reached.
    1738             :          * During application shutdown, bdevs automatically get unregistered by the bdev layer
    1739             :          * so this is the only way currently to do this correctly.
    1740             :          * TODO: try to handle this correctly in bdev layer instead.
    1741             :          */
    1742          14 :         rc = spdk_bdev_open_ext(raid_bdev_gen->name, false, raid_bdev_event_cb, raid_bdev,
    1743             :                                 &raid_bdev->self_desc);
    1744          14 :         if (rc != 0) {
    1745           0 :                 SPDK_ERRLOG("Failed to open raid bdev '%s': %s\n",
    1746             :                             raid_bdev_gen->name, spdk_strerror(-rc));
    1747           0 :                 spdk_bdev_unregister(raid_bdev_gen, NULL, NULL);
    1748           0 :                 goto out;
    1749             :         }
    1750             : 
    1751          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev generic %p\n", raid_bdev_gen);
    1752          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n",
    1753             :                       raid_bdev_gen->name, raid_bdev);
    1754          14 : out:
    1755          14 :         if (rc != 0) {
    1756           0 :                 if (raid_bdev->module->stop != NULL) {
    1757           0 :                         raid_bdev->module->stop(raid_bdev);
    1758             :                 }
    1759           0 :                 spdk_io_device_unregister(raid_bdev, NULL);
    1760           0 :                 raid_bdev->state = RAID_BDEV_STATE_CONFIGURING;
    1761             :         }
    1762             : 
    1763          14 :         if (raid_bdev->configure_cb != NULL) {
    1764          14 :                 raid_bdev->configure_cb(raid_bdev->configure_cb_ctx, rc);
    1765          14 :                 raid_bdev->configure_cb = NULL;
    1766             :         }
    1767          14 : }
    1768             : 
    1769             : static void
    1770           1 : raid_bdev_configure_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    1771             : {
    1772           1 :         if (status == 0) {
    1773           1 :                 raid_bdev_configure_cont(raid_bdev);
    1774             :         } else {
    1775           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock: %s\n",
    1776             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    1777           0 :                 if (raid_bdev->module->stop != NULL) {
    1778           0 :                         raid_bdev->module->stop(raid_bdev);
    1779             :                 }
    1780           0 :                 if (raid_bdev->configure_cb != NULL) {
    1781           0 :                         raid_bdev->configure_cb(raid_bdev->configure_cb_ctx, status);
    1782           0 :                         raid_bdev->configure_cb = NULL;
    1783             :                 }
    1784             :         }
    1785           1 : }
    1786             : 
    1787             : /*
    1788             :  * brief:
    1789             :  * If raid bdev config is complete, then only register the raid bdev to
    1790             :  * bdev layer and remove this raid bdev from configuring list and
    1791             :  * insert the raid bdev to configured list
    1792             :  * params:
    1793             :  * raid_bdev - pointer to raid bdev
    1794             :  * returns:
    1795             :  * 0 - success
    1796             :  * non zero - failure
    1797             :  */
    1798             : static int
    1799          14 : raid_bdev_configure(struct raid_bdev *raid_bdev, raid_bdev_configure_cb cb, void *cb_ctx)
    1800             : {
    1801          14 :         uint32_t data_block_size = spdk_bdev_get_data_block_size(&raid_bdev->bdev);
    1802             :         int rc;
    1803             : 
    1804          14 :         assert(raid_bdev->state == RAID_BDEV_STATE_CONFIGURING);
    1805          14 :         assert(raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational);
    1806          14 :         assert(raid_bdev->bdev.blocklen > 0);
    1807             : 
    1808             :         /* The strip_size_kb is read in from user in KB. Convert to blocks here for
    1809             :          * internal use.
    1810             :          */
    1811          14 :         raid_bdev->strip_size = (raid_bdev->strip_size_kb * 1024) / data_block_size;
    1812          14 :         if (raid_bdev->strip_size == 0 && raid_bdev->level != RAID1) {
    1813           0 :                 SPDK_ERRLOG("Strip size cannot be smaller than the device block size\n");
    1814           0 :                 return -EINVAL;
    1815             :         }
    1816          14 :         raid_bdev->strip_size_shift = spdk_u32log2(raid_bdev->strip_size);
    1817             : 
    1818          14 :         rc = raid_bdev->module->start(raid_bdev);
    1819          14 :         if (rc != 0) {
    1820           0 :                 SPDK_ERRLOG("raid module startup callback failed\n");
    1821           0 :                 return rc;
    1822             :         }
    1823             : 
    1824          14 :         assert(raid_bdev->configure_cb == NULL);
    1825          14 :         raid_bdev->configure_cb = cb;
    1826          14 :         raid_bdev->configure_cb_ctx = cb_ctx;
    1827             : 
    1828          14 :         if (raid_bdev->superblock_enabled) {
    1829           1 :                 if (raid_bdev->sb == NULL) {
    1830           1 :                         rc = raid_bdev_alloc_superblock(raid_bdev, data_block_size);
    1831           1 :                         if (rc == 0) {
    1832           1 :                                 raid_bdev_init_superblock(raid_bdev);
    1833             :                         }
    1834             :                 } else {
    1835           0 :                         assert(spdk_uuid_compare(&raid_bdev->sb->uuid, &raid_bdev->bdev.uuid) == 0);
    1836           0 :                         if (raid_bdev->sb->block_size != data_block_size) {
    1837           0 :                                 SPDK_ERRLOG("blocklen does not match value in superblock\n");
    1838           0 :                                 rc = -EINVAL;
    1839             :                         }
    1840           0 :                         if (raid_bdev->sb->raid_size != raid_bdev->bdev.blockcnt) {
    1841           0 :                                 SPDK_ERRLOG("blockcnt does not match value in superblock\n");
    1842           0 :                                 rc = -EINVAL;
    1843             :                         }
    1844             :                 }
    1845             : 
    1846           1 :                 if (rc != 0) {
    1847           0 :                         raid_bdev->configure_cb = NULL;
    1848           0 :                         if (raid_bdev->module->stop != NULL) {
    1849           0 :                                 raid_bdev->module->stop(raid_bdev);
    1850             :                         }
    1851           0 :                         return rc;
    1852             :                 }
    1853             : 
    1854           1 :                 raid_bdev_write_superblock(raid_bdev, raid_bdev_configure_write_sb_cb, NULL);
    1855             :         } else {
    1856          13 :                 raid_bdev_configure_cont(raid_bdev);
    1857             :         }
    1858             : 
    1859          14 :         return 0;
    1860             : }
    1861             : 
    1862             : /*
    1863             :  * brief:
    1864             :  * If raid bdev is online and registered, change the bdev state to
    1865             :  * configuring and unregister this raid device. Queue this raid device
    1866             :  * in configuring list
    1867             :  * params:
    1868             :  * raid_bdev - pointer to raid bdev
    1869             :  * cb_fn - callback function
    1870             :  * cb_arg - argument to callback function
    1871             :  * returns:
    1872             :  * none
    1873             :  */
    1874             : static void
    1875          14 : raid_bdev_deconfigure(struct raid_bdev *raid_bdev, raid_bdev_destruct_cb cb_fn,
    1876             :                       void *cb_arg)
    1877             : {
    1878          14 :         if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    1879           0 :                 if (cb_fn) {
    1880           0 :                         cb_fn(cb_arg, 0);
    1881             :                 }
    1882           0 :                 return;
    1883             :         }
    1884             : 
    1885          14 :         raid_bdev->state = RAID_BDEV_STATE_OFFLINE;
    1886          14 :         SPDK_DEBUGLOG(bdev_raid, "raid bdev state changing from online to offline\n");
    1887             : 
    1888          14 :         spdk_bdev_unregister(&raid_bdev->bdev, cb_fn, cb_arg);
    1889             : }
    1890             : 
    1891             : /*
    1892             :  * brief:
    1893             :  * raid_bdev_find_base_info_by_bdev function finds the base bdev info by bdev.
    1894             :  * params:
    1895             :  * base_bdev - pointer to base bdev
    1896             :  * returns:
    1897             :  * base bdev info if found, otherwise NULL.
    1898             :  */
    1899             : static struct raid_base_bdev_info *
    1900           0 : raid_bdev_find_base_info_by_bdev(struct spdk_bdev *base_bdev)
    1901             : {
    1902             :         struct raid_bdev *raid_bdev;
    1903             :         struct raid_base_bdev_info *base_info;
    1904             : 
    1905           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    1906           0 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    1907           0 :                         if (base_info->desc != NULL &&
    1908           0 :                             spdk_bdev_desc_get_bdev(base_info->desc) == base_bdev) {
    1909           0 :                                 return base_info;
    1910             :                         }
    1911             :                 }
    1912             :         }
    1913             : 
    1914           0 :         return NULL;
    1915             : }
    1916             : 
    1917             : static void
    1918           0 : raid_bdev_remove_base_bdev_done(struct raid_base_bdev_info *base_info, int status)
    1919             : {
    1920           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1921             : 
    1922           0 :         assert(base_info->remove_scheduled);
    1923           0 :         base_info->remove_scheduled = false;
    1924             : 
    1925           0 :         if (status == 0) {
    1926           0 :                 raid_bdev->num_base_bdevs_operational--;
    1927           0 :                 if (raid_bdev->num_base_bdevs_operational < raid_bdev->min_base_bdevs_operational) {
    1928             :                         /* There is not enough base bdevs to keep the raid bdev operational. */
    1929           0 :                         raid_bdev_deconfigure(raid_bdev, base_info->remove_cb, base_info->remove_cb_ctx);
    1930           0 :                         return;
    1931             :                 }
    1932             :         }
    1933             : 
    1934           0 :         if (base_info->remove_cb != NULL) {
    1935           0 :                 base_info->remove_cb(base_info->remove_cb_ctx, status);
    1936             :         }
    1937             : }
    1938             : 
    1939             : static void
    1940           0 : raid_bdev_remove_base_bdev_on_unquiesced(void *ctx, int status)
    1941             : {
    1942           0 :         struct raid_base_bdev_info *base_info = ctx;
    1943           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1944             : 
    1945           0 :         if (status != 0) {
    1946           0 :                 SPDK_ERRLOG("Failed to unquiesce raid bdev %s: %s\n",
    1947             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    1948             :         }
    1949             : 
    1950           0 :         raid_bdev_remove_base_bdev_done(base_info, status);
    1951           0 : }
    1952             : 
    1953             : static void
    1954           0 : raid_bdev_channel_remove_base_bdev(struct spdk_io_channel_iter *i)
    1955             : {
    1956           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    1957           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    1958           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    1959           0 :         uint8_t idx = raid_bdev_base_bdev_slot(base_info);
    1960             : 
    1961           0 :         SPDK_DEBUGLOG(bdev_raid, "slot: %u raid_ch: %p\n", idx, raid_ch);
    1962             : 
    1963           0 :         if (raid_ch->base_channel[idx] != NULL) {
    1964           0 :                 spdk_put_io_channel(raid_ch->base_channel[idx]);
    1965           0 :                 raid_ch->base_channel[idx] = NULL;
    1966             :         }
    1967             : 
    1968           0 :         if (raid_ch->process.ch_processed != NULL) {
    1969           0 :                 raid_ch->process.ch_processed->base_channel[idx] = NULL;
    1970             :         }
    1971             : 
    1972           0 :         spdk_for_each_channel_continue(i, 0);
    1973           0 : }
    1974             : 
    1975             : static void
    1976           0 : raid_bdev_channels_remove_base_bdev_done(struct spdk_io_channel_iter *i, int status)
    1977             : {
    1978           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    1979           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    1980             : 
    1981           0 :         raid_bdev_free_base_bdev_resource(base_info);
    1982             : 
    1983           0 :         spdk_bdev_unquiesce(&raid_bdev->bdev, &g_raid_if, raid_bdev_remove_base_bdev_on_unquiesced,
    1984             :                             base_info);
    1985           0 : }
    1986             : 
    1987             : static void
    1988           0 : raid_bdev_remove_base_bdev_cont(struct raid_base_bdev_info *base_info)
    1989             : {
    1990           0 :         raid_bdev_deconfigure_base_bdev(base_info);
    1991             : 
    1992           0 :         spdk_for_each_channel(base_info->raid_bdev, raid_bdev_channel_remove_base_bdev, base_info,
    1993             :                               raid_bdev_channels_remove_base_bdev_done);
    1994           0 : }
    1995             : 
    1996             : static void
    1997           0 : raid_bdev_remove_base_bdev_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    1998             : {
    1999           0 :         struct raid_base_bdev_info *base_info = ctx;
    2000             : 
    2001           0 :         if (status != 0) {
    2002           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock: %s\n",
    2003             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2004           0 :                 raid_bdev_remove_base_bdev_done(base_info, status);
    2005           0 :                 return;
    2006             :         }
    2007             : 
    2008           0 :         raid_bdev_remove_base_bdev_cont(base_info);
    2009             : }
    2010             : 
    2011             : static void
    2012           0 : raid_bdev_remove_base_bdev_on_quiesced(void *ctx, int status)
    2013             : {
    2014           0 :         struct raid_base_bdev_info *base_info = ctx;
    2015           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    2016             : 
    2017           0 :         if (status != 0) {
    2018           0 :                 SPDK_ERRLOG("Failed to quiesce raid bdev %s: %s\n",
    2019             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2020           0 :                 raid_bdev_remove_base_bdev_done(base_info, status);
    2021           0 :                 return;
    2022             :         }
    2023             : 
    2024           0 :         if (raid_bdev->sb) {
    2025           0 :                 struct raid_bdev_superblock *sb = raid_bdev->sb;
    2026           0 :                 uint8_t slot = raid_bdev_base_bdev_slot(base_info);
    2027             :                 uint8_t i;
    2028             : 
    2029           0 :                 for (i = 0; i < sb->base_bdevs_size; i++) {
    2030           0 :                         struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    2031             : 
    2032           0 :                         if (sb_base_bdev->state == RAID_SB_BASE_BDEV_CONFIGURED &&
    2033           0 :                             sb_base_bdev->slot == slot) {
    2034           0 :                                 if (base_info->is_failed) {
    2035           0 :                                         sb_base_bdev->state = RAID_SB_BASE_BDEV_FAILED;
    2036             :                                 } else {
    2037           0 :                                         sb_base_bdev->state = RAID_SB_BASE_BDEV_MISSING;
    2038             :                                 }
    2039             : 
    2040           0 :                                 raid_bdev_write_superblock(raid_bdev, raid_bdev_remove_base_bdev_write_sb_cb, base_info);
    2041           0 :                                 return;
    2042             :                         }
    2043             :                 }
    2044             :         }
    2045             : 
    2046           0 :         raid_bdev_remove_base_bdev_cont(base_info);
    2047             : }
    2048             : 
    2049             : static int
    2050           0 : raid_bdev_remove_base_bdev_quiesce(struct raid_base_bdev_info *base_info)
    2051             : {
    2052           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2053             : 
    2054           0 :         return spdk_bdev_quiesce(&base_info->raid_bdev->bdev, &g_raid_if,
    2055             :                                  raid_bdev_remove_base_bdev_on_quiesced, base_info);
    2056             : }
    2057             : 
    2058             : struct raid_bdev_process_base_bdev_remove_ctx {
    2059             :         struct raid_bdev_process *process;
    2060             :         struct raid_base_bdev_info *base_info;
    2061             :         uint8_t num_base_bdevs_operational;
    2062             : };
    2063             : 
    2064             : static void
    2065           0 : _raid_bdev_process_base_bdev_remove_cont(void *ctx)
    2066             : {
    2067           0 :         struct raid_base_bdev_info *base_info = ctx;
    2068             :         int ret;
    2069             : 
    2070           0 :         ret = raid_bdev_remove_base_bdev_quiesce(base_info);
    2071           0 :         if (ret != 0) {
    2072           0 :                 raid_bdev_remove_base_bdev_done(base_info, ret);
    2073             :         }
    2074           0 : }
    2075             : 
    2076             : static void
    2077           0 : raid_bdev_process_base_bdev_remove_cont(void *_ctx)
    2078             : {
    2079           0 :         struct raid_bdev_process_base_bdev_remove_ctx *ctx = _ctx;
    2080           0 :         struct raid_base_bdev_info *base_info = ctx->base_info;
    2081             : 
    2082           0 :         free(ctx);
    2083             : 
    2084           0 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), _raid_bdev_process_base_bdev_remove_cont,
    2085             :                              base_info);
    2086           0 : }
    2087             : 
    2088             : static void
    2089           0 : _raid_bdev_process_base_bdev_remove(void *_ctx)
    2090             : {
    2091           0 :         struct raid_bdev_process_base_bdev_remove_ctx *ctx = _ctx;
    2092           0 :         struct raid_bdev_process *process = ctx->process;
    2093             :         int ret;
    2094             : 
    2095           0 :         if (ctx->base_info != process->target &&
    2096           0 :             ctx->num_base_bdevs_operational > process->raid_bdev->min_base_bdevs_operational) {
    2097             :                 /* process doesn't need to be stopped */
    2098           0 :                 raid_bdev_process_base_bdev_remove_cont(ctx);
    2099           0 :                 return;
    2100             :         }
    2101             : 
    2102           0 :         assert(process->state > RAID_PROCESS_STATE_INIT &&
    2103             :                process->state < RAID_PROCESS_STATE_STOPPED);
    2104             : 
    2105           0 :         ret = raid_bdev_process_add_finish_action(process, raid_bdev_process_base_bdev_remove_cont, ctx);
    2106           0 :         if (ret != 0) {
    2107           0 :                 raid_bdev_remove_base_bdev_done(ctx->base_info, ret);
    2108           0 :                 free(ctx);
    2109           0 :                 return;
    2110             :         }
    2111             : 
    2112           0 :         process->state = RAID_PROCESS_STATE_STOPPING;
    2113             : 
    2114           0 :         if (process->status == 0) {
    2115           0 :                 process->status = -ENODEV;
    2116             :         }
    2117             : }
    2118             : 
    2119             : static int
    2120           0 : raid_bdev_process_base_bdev_remove(struct raid_bdev_process *process,
    2121             :                                    struct raid_base_bdev_info *base_info)
    2122             : {
    2123             :         struct raid_bdev_process_base_bdev_remove_ctx *ctx;
    2124             : 
    2125           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2126             : 
    2127           0 :         ctx = calloc(1, sizeof(*ctx));
    2128           0 :         if (ctx == NULL) {
    2129           0 :                 return -ENOMEM;
    2130             :         }
    2131             : 
    2132             :         /*
    2133             :          * We have to send the process and num_base_bdevs_operational in the message ctx
    2134             :          * because the process thread should not access raid_bdev's properties. Particularly,
    2135             :          * raid_bdev->process may be cleared by the time the message is handled, but ctx->process
    2136             :          * will still be valid until the process is fully stopped.
    2137             :          */
    2138           0 :         ctx->base_info = base_info;
    2139           0 :         ctx->process = process;
    2140             :         /*
    2141             :          * raid_bdev->num_base_bdevs_operational can't be used here because it is decremented
    2142             :          * after the removal and more than one base bdev may be removed at the same time
    2143             :          */
    2144           0 :         RAID_FOR_EACH_BASE_BDEV(process->raid_bdev, base_info) {
    2145           0 :                 if (base_info->is_configured && !base_info->remove_scheduled) {
    2146           0 :                         ctx->num_base_bdevs_operational++;
    2147             :                 }
    2148             :         }
    2149             : 
    2150           0 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_base_bdev_remove, ctx);
    2151             : 
    2152           0 :         return 0;
    2153             : }
    2154             : 
    2155             : static int
    2156           0 : _raid_bdev_remove_base_bdev(struct raid_base_bdev_info *base_info,
    2157             :                             raid_base_bdev_cb cb_fn, void *cb_ctx)
    2158             : {
    2159           0 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    2160           0 :         int ret = 0;
    2161             : 
    2162           0 :         SPDK_DEBUGLOG(bdev_raid, "%s\n", base_info->name);
    2163             : 
    2164           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2165             : 
    2166           0 :         if (base_info->remove_scheduled || !base_info->is_configured) {
    2167           0 :                 return -ENODEV;
    2168             :         }
    2169             : 
    2170           0 :         assert(base_info->desc);
    2171           0 :         base_info->remove_scheduled = true;
    2172             : 
    2173           0 :         if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    2174             :                 /*
    2175             :                  * As raid bdev is not registered yet or already unregistered,
    2176             :                  * so cleanup should be done here itself.
    2177             :                  *
    2178             :                  * Removing a base bdev at this stage does not change the number of operational
    2179             :                  * base bdevs, only the number of discovered base bdevs.
    2180             :                  */
    2181           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    2182           0 :                 base_info->remove_scheduled = false;
    2183           0 :                 if (raid_bdev->num_base_bdevs_discovered == 0 &&
    2184           0 :                     raid_bdev->state == RAID_BDEV_STATE_OFFLINE) {
    2185             :                         /* There is no base bdev for this raid, so free the raid device. */
    2186           0 :                         raid_bdev_cleanup_and_free(raid_bdev);
    2187             :                 }
    2188           0 :                 if (cb_fn != NULL) {
    2189           0 :                         cb_fn(cb_ctx, 0);
    2190             :                 }
    2191           0 :         } else if (raid_bdev->min_base_bdevs_operational == raid_bdev->num_base_bdevs) {
    2192             :                 /* This raid bdev does not tolerate removing a base bdev. */
    2193           0 :                 raid_bdev->num_base_bdevs_operational--;
    2194           0 :                 raid_bdev_deconfigure(raid_bdev, cb_fn, cb_ctx);
    2195             :         } else {
    2196           0 :                 base_info->remove_cb = cb_fn;
    2197           0 :                 base_info->remove_cb_ctx = cb_ctx;
    2198             : 
    2199           0 :                 if (raid_bdev->process != NULL) {
    2200           0 :                         ret = raid_bdev_process_base_bdev_remove(raid_bdev->process, base_info);
    2201             :                 } else {
    2202           0 :                         ret = raid_bdev_remove_base_bdev_quiesce(base_info);
    2203             :                 }
    2204             : 
    2205           0 :                 if (ret != 0) {
    2206           0 :                         base_info->remove_scheduled = false;
    2207             :                 }
    2208             :         }
    2209             : 
    2210           0 :         return ret;
    2211             : }
    2212             : 
    2213             : /*
    2214             :  * brief:
    2215             :  * raid_bdev_remove_base_bdev function is called by below layers when base_bdev
    2216             :  * is removed. This function checks if this base bdev is part of any raid bdev
    2217             :  * or not. If yes, it takes necessary action on that particular raid bdev.
    2218             :  * params:
    2219             :  * base_bdev - pointer to base bdev which got removed
    2220             :  * cb_fn - callback function
    2221             :  * cb_arg - argument to callback function
    2222             :  * returns:
    2223             :  * 0 - success
    2224             :  * non zero - failure
    2225             :  */
    2226             : int
    2227           0 : raid_bdev_remove_base_bdev(struct spdk_bdev *base_bdev, raid_base_bdev_cb cb_fn, void *cb_ctx)
    2228             : {
    2229             :         struct raid_base_bdev_info *base_info;
    2230             : 
    2231             :         /* Find the raid_bdev which has claimed this base_bdev */
    2232           0 :         base_info = raid_bdev_find_base_info_by_bdev(base_bdev);
    2233           0 :         if (!base_info) {
    2234           0 :                 SPDK_ERRLOG("bdev to remove '%s' not found\n", base_bdev->name);
    2235           0 :                 return -ENODEV;
    2236             :         }
    2237             : 
    2238           0 :         return _raid_bdev_remove_base_bdev(base_info, cb_fn, cb_ctx);
    2239             : }
    2240             : 
    2241             : static void
    2242           0 : raid_bdev_fail_base_remove_cb(void *ctx, int status)
    2243             : {
    2244           0 :         struct raid_base_bdev_info *base_info = ctx;
    2245             : 
    2246           0 :         if (status != 0) {
    2247           0 :                 SPDK_WARNLOG("Failed to remove base bdev %s\n", base_info->name);
    2248           0 :                 base_info->is_failed = false;
    2249             :         }
    2250           0 : }
    2251             : 
    2252             : static void
    2253           0 : _raid_bdev_fail_base_bdev(void *ctx)
    2254             : {
    2255           0 :         struct raid_base_bdev_info *base_info = ctx;
    2256             :         int rc;
    2257             : 
    2258           0 :         if (base_info->is_failed) {
    2259           0 :                 return;
    2260             :         }
    2261           0 :         base_info->is_failed = true;
    2262             : 
    2263           0 :         SPDK_NOTICELOG("Failing base bdev in slot %d ('%s') of raid bdev '%s'\n",
    2264             :                        raid_bdev_base_bdev_slot(base_info), base_info->name, base_info->raid_bdev->bdev.name);
    2265             : 
    2266           0 :         rc = _raid_bdev_remove_base_bdev(base_info, raid_bdev_fail_base_remove_cb, base_info);
    2267           0 :         if (rc != 0) {
    2268           0 :                 raid_bdev_fail_base_remove_cb(base_info, rc);
    2269             :         }
    2270             : }
    2271             : 
    2272             : void
    2273           0 : raid_bdev_fail_base_bdev(struct raid_base_bdev_info *base_info)
    2274             : {
    2275           0 :         spdk_thread_exec_msg(spdk_thread_get_app_thread(), _raid_bdev_fail_base_bdev, base_info);
    2276           0 : }
    2277             : 
    2278             : static void
    2279           0 : raid_bdev_resize_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    2280             : {
    2281           0 :         if (status != 0) {
    2282           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock after resizing the bdev: %s\n",
    2283             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2284             :         }
    2285           0 : }
    2286             : 
    2287             : /*
    2288             :  * brief:
    2289             :  * raid_bdev_resize_base_bdev function is called by below layers when base_bdev
    2290             :  * is resized. This function checks if the smallest size of the base_bdevs is changed.
    2291             :  * If yes, call module handler to resize the raid_bdev if implemented.
    2292             :  * params:
    2293             :  * base_bdev - pointer to base bdev which got resized.
    2294             :  * returns:
    2295             :  * none
    2296             :  */
    2297             : static void
    2298           0 : raid_bdev_resize_base_bdev(struct spdk_bdev *base_bdev)
    2299             : {
    2300             :         struct raid_bdev *raid_bdev;
    2301             :         struct raid_base_bdev_info *base_info;
    2302             :         uint64_t blockcnt_old;
    2303             : 
    2304           0 :         SPDK_DEBUGLOG(bdev_raid, "raid_bdev_resize_base_bdev\n");
    2305             : 
    2306           0 :         base_info = raid_bdev_find_base_info_by_bdev(base_bdev);
    2307             : 
    2308             :         /* Find the raid_bdev which has claimed this base_bdev */
    2309           0 :         if (!base_info) {
    2310           0 :                 SPDK_ERRLOG("raid_bdev whose base_bdev '%s' not found\n", base_bdev->name);
    2311           0 :                 return;
    2312             :         }
    2313           0 :         raid_bdev = base_info->raid_bdev;
    2314             : 
    2315           0 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    2316             : 
    2317           0 :         SPDK_NOTICELOG("base_bdev '%s' was resized: old size %" PRIu64 ", new size %" PRIu64 "\n",
    2318             :                        base_bdev->name, base_info->blockcnt, base_bdev->blockcnt);
    2319             : 
    2320           0 :         base_info->blockcnt = base_bdev->blockcnt;
    2321             : 
    2322           0 :         if (!raid_bdev->module->resize) {
    2323           0 :                 return;
    2324             :         }
    2325             : 
    2326           0 :         blockcnt_old = raid_bdev->bdev.blockcnt;
    2327           0 :         if (raid_bdev->module->resize(raid_bdev) == false) {
    2328           0 :                 return;
    2329             :         }
    2330             : 
    2331           0 :         SPDK_NOTICELOG("raid bdev '%s': block count was changed from %" PRIu64 " to %" PRIu64 "\n",
    2332             :                        raid_bdev->bdev.name, blockcnt_old, raid_bdev->bdev.blockcnt);
    2333             : 
    2334           0 :         if (raid_bdev->superblock_enabled) {
    2335           0 :                 struct raid_bdev_superblock *sb = raid_bdev->sb;
    2336             :                 uint8_t i;
    2337             : 
    2338           0 :                 for (i = 0; i < sb->base_bdevs_size; i++) {
    2339           0 :                         struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    2340             : 
    2341           0 :                         if (sb_base_bdev->slot < raid_bdev->num_base_bdevs) {
    2342           0 :                                 base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    2343           0 :                                 sb_base_bdev->data_size = base_info->data_size;
    2344             :                         }
    2345             :                 }
    2346           0 :                 sb->raid_size = raid_bdev->bdev.blockcnt;
    2347           0 :                 raid_bdev_write_superblock(raid_bdev, raid_bdev_resize_write_sb_cb, NULL);
    2348             :         }
    2349             : }
    2350             : 
    2351             : /*
    2352             :  * brief:
    2353             :  * raid_bdev_event_base_bdev function is called by below layers when base_bdev
    2354             :  * triggers asynchronous event.
    2355             :  * params:
    2356             :  * type - event details.
    2357             :  * bdev - bdev that triggered event.
    2358             :  * event_ctx - context for event.
    2359             :  * returns:
    2360             :  * none
    2361             :  */
    2362             : static void
    2363           0 : raid_bdev_event_base_bdev(enum spdk_bdev_event_type type, struct spdk_bdev *bdev,
    2364             :                           void *event_ctx)
    2365             : {
    2366             :         int rc;
    2367             : 
    2368           0 :         switch (type) {
    2369           0 :         case SPDK_BDEV_EVENT_REMOVE:
    2370           0 :                 rc = raid_bdev_remove_base_bdev(bdev, NULL, NULL);
    2371           0 :                 if (rc != 0) {
    2372           0 :                         SPDK_ERRLOG("Failed to remove base bdev %s: %s\n",
    2373             :                                     spdk_bdev_get_name(bdev), spdk_strerror(-rc));
    2374             :                 }
    2375           0 :                 break;
    2376           0 :         case SPDK_BDEV_EVENT_RESIZE:
    2377           0 :                 raid_bdev_resize_base_bdev(bdev);
    2378           0 :                 break;
    2379           0 :         default:
    2380           0 :                 SPDK_NOTICELOG("Unsupported bdev event: type %d\n", type);
    2381           0 :                 break;
    2382             :         }
    2383           0 : }
    2384             : 
    2385             : /*
    2386             :  * brief:
    2387             :  * Deletes the specified raid bdev
    2388             :  * params:
    2389             :  * raid_bdev - pointer to raid bdev
    2390             :  * cb_fn - callback function
    2391             :  * cb_arg - argument to callback function
    2392             :  */
    2393             : void
    2394          16 : raid_bdev_delete(struct raid_bdev *raid_bdev, raid_bdev_destruct_cb cb_fn, void *cb_arg)
    2395             : {
    2396             :         struct raid_base_bdev_info *base_info;
    2397             : 
    2398          16 :         SPDK_DEBUGLOG(bdev_raid, "delete raid bdev: %s\n", raid_bdev->bdev.name);
    2399             : 
    2400          16 :         if (raid_bdev->destroy_started) {
    2401           0 :                 SPDK_DEBUGLOG(bdev_raid, "destroying raid bdev %s is already started\n",
    2402             :                               raid_bdev->bdev.name);
    2403           0 :                 if (cb_fn) {
    2404           0 :                         cb_fn(cb_arg, -EALREADY);
    2405             :                 }
    2406           0 :                 return;
    2407             :         }
    2408             : 
    2409          16 :         raid_bdev->destroy_started = true;
    2410             : 
    2411         528 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    2412         512 :                 base_info->remove_scheduled = true;
    2413             : 
    2414         512 :                 if (raid_bdev->state != RAID_BDEV_STATE_ONLINE) {
    2415             :                         /*
    2416             :                          * As raid bdev is not registered yet or already unregistered,
    2417             :                          * so cleanup should be done here itself.
    2418             :                          */
    2419          64 :                         raid_bdev_free_base_bdev_resource(base_info);
    2420             :                 }
    2421             :         }
    2422             : 
    2423          16 :         if (raid_bdev->num_base_bdevs_discovered == 0) {
    2424             :                 /* There is no base bdev for this raid, so free the raid device. */
    2425           2 :                 raid_bdev_cleanup_and_free(raid_bdev);
    2426           2 :                 if (cb_fn) {
    2427           0 :                         cb_fn(cb_arg, 0);
    2428             :                 }
    2429             :         } else {
    2430          14 :                 raid_bdev_deconfigure(raid_bdev, cb_fn, cb_arg);
    2431             :         }
    2432             : }
    2433             : 
    2434             : static void
    2435           0 : raid_bdev_process_finish_write_sb_cb(int status, struct raid_bdev *raid_bdev, void *ctx)
    2436             : {
    2437           0 :         if (status != 0) {
    2438           0 :                 SPDK_ERRLOG("Failed to write raid bdev '%s' superblock after background process finished: %s\n",
    2439             :                             raid_bdev->bdev.name, spdk_strerror(-status));
    2440             :         }
    2441           0 : }
    2442             : 
    2443             : static void
    2444           0 : raid_bdev_process_finish_write_sb(void *ctx)
    2445             : {
    2446           0 :         struct raid_bdev *raid_bdev = ctx;
    2447           0 :         struct raid_bdev_superblock *sb = raid_bdev->sb;
    2448             :         struct raid_bdev_sb_base_bdev *sb_base_bdev;
    2449             :         struct raid_base_bdev_info *base_info;
    2450             :         uint8_t i;
    2451             : 
    2452           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    2453           0 :                 sb_base_bdev = &sb->base_bdevs[i];
    2454             : 
    2455           0 :                 if (sb_base_bdev->state != RAID_SB_BASE_BDEV_CONFIGURED &&
    2456           0 :                     sb_base_bdev->slot < raid_bdev->num_base_bdevs) {
    2457           0 :                         base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    2458           0 :                         if (base_info->is_configured) {
    2459           0 :                                 sb_base_bdev->state = RAID_SB_BASE_BDEV_CONFIGURED;
    2460           0 :                                 spdk_uuid_copy(&sb_base_bdev->uuid, &base_info->uuid);
    2461             :                         }
    2462             :                 }
    2463             :         }
    2464             : 
    2465           0 :         raid_bdev_write_superblock(raid_bdev, raid_bdev_process_finish_write_sb_cb, NULL);
    2466           0 : }
    2467             : 
    2468             : static void raid_bdev_process_free(struct raid_bdev_process *process);
    2469             : 
    2470             : static void
    2471           2 : _raid_bdev_process_finish_done(void *ctx)
    2472             : {
    2473           2 :         struct raid_bdev_process *process = ctx;
    2474             :         struct raid_process_finish_action *finish_action;
    2475             : 
    2476           2 :         while ((finish_action = TAILQ_FIRST(&process->finish_actions)) != NULL) {
    2477           0 :                 TAILQ_REMOVE(&process->finish_actions, finish_action, link);
    2478           0 :                 finish_action->cb(finish_action->cb_ctx);
    2479           0 :                 free(finish_action);
    2480             :         }
    2481             : 
    2482           2 :         spdk_poller_unregister(&process->qos.process_continue_poller);
    2483             : 
    2484           2 :         raid_bdev_process_free(process);
    2485             : 
    2486           2 :         spdk_thread_exit(spdk_get_thread());
    2487           2 : }
    2488             : 
    2489             : static void
    2490           0 : raid_bdev_process_finish_target_removed(void *ctx, int status)
    2491             : {
    2492           0 :         struct raid_bdev_process *process = ctx;
    2493             : 
    2494           0 :         if (status != 0) {
    2495           0 :                 SPDK_ERRLOG("Failed to remove target bdev: %s\n", spdk_strerror(-status));
    2496             :         }
    2497             : 
    2498           0 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_finish_done, process);
    2499           0 : }
    2500             : 
    2501             : static void
    2502           2 : raid_bdev_process_finish_unquiesced(void *ctx, int status)
    2503             : {
    2504           2 :         struct raid_bdev_process *process = ctx;
    2505             : 
    2506           2 :         if (status != 0) {
    2507           0 :                 SPDK_ERRLOG("Failed to unquiesce bdev: %s\n", spdk_strerror(-status));
    2508             :         }
    2509             : 
    2510           2 :         if (process->status != 0) {
    2511           0 :                 status = _raid_bdev_remove_base_bdev(process->target, raid_bdev_process_finish_target_removed,
    2512             :                                                      process);
    2513           0 :                 if (status != 0) {
    2514           0 :                         raid_bdev_process_finish_target_removed(process, status);
    2515             :                 }
    2516           0 :                 return;
    2517             :         }
    2518             : 
    2519           2 :         spdk_thread_send_msg(process->thread, _raid_bdev_process_finish_done, process);
    2520             : }
    2521             : 
    2522             : static void
    2523           2 : raid_bdev_process_finish_unquiesce(void *ctx)
    2524             : {
    2525           2 :         struct raid_bdev_process *process = ctx;
    2526             :         int rc;
    2527             : 
    2528           2 :         rc = spdk_bdev_unquiesce(&process->raid_bdev->bdev, &g_raid_if,
    2529             :                                  raid_bdev_process_finish_unquiesced, process);
    2530           2 :         if (rc != 0) {
    2531           0 :                 raid_bdev_process_finish_unquiesced(process, rc);
    2532             :         }
    2533           2 : }
    2534             : 
    2535             : static void
    2536           2 : raid_bdev_process_finish_done(void *ctx)
    2537             : {
    2538           2 :         struct raid_bdev_process *process = ctx;
    2539           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2540             : 
    2541           2 :         if (process->raid_ch != NULL) {
    2542           2 :                 spdk_put_io_channel(spdk_io_channel_from_ctx(process->raid_ch));
    2543             :         }
    2544             : 
    2545           2 :         process->state = RAID_PROCESS_STATE_STOPPED;
    2546             : 
    2547           2 :         if (process->status == 0) {
    2548           2 :                 SPDK_NOTICELOG("Finished %s on raid bdev %s\n",
    2549             :                                raid_bdev_process_to_str(process->type),
    2550             :                                raid_bdev->bdev.name);
    2551           2 :                 if (raid_bdev->superblock_enabled) {
    2552           0 :                         spdk_thread_send_msg(spdk_thread_get_app_thread(),
    2553             :                                              raid_bdev_process_finish_write_sb,
    2554             :                                              raid_bdev);
    2555             :                 }
    2556             :         } else {
    2557           0 :                 SPDK_WARNLOG("Finished %s on raid bdev %s: %s\n",
    2558             :                              raid_bdev_process_to_str(process->type),
    2559             :                              raid_bdev->bdev.name,
    2560             :                              spdk_strerror(-process->status));
    2561             :         }
    2562             : 
    2563           2 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), raid_bdev_process_finish_unquiesce,
    2564             :                              process);
    2565           2 : }
    2566             : 
    2567             : static void
    2568           2 : __raid_bdev_process_finish(struct spdk_io_channel_iter *i, int status)
    2569             : {
    2570           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2571             : 
    2572           2 :         spdk_thread_send_msg(process->thread, raid_bdev_process_finish_done, process);
    2573           2 : }
    2574             : 
    2575             : static void
    2576           2 : raid_bdev_channel_process_finish(struct spdk_io_channel_iter *i)
    2577             : {
    2578           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2579           2 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2580           2 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2581             : 
    2582           2 :         if (process->status == 0) {
    2583           2 :                 uint8_t slot = raid_bdev_base_bdev_slot(process->target);
    2584             : 
    2585           2 :                 raid_ch->base_channel[slot] = raid_ch->process.target_ch;
    2586           2 :                 raid_ch->process.target_ch = NULL;
    2587             :         }
    2588             : 
    2589           2 :         raid_bdev_ch_process_cleanup(raid_ch);
    2590             : 
    2591           2 :         spdk_for_each_channel_continue(i, 0);
    2592           2 : }
    2593             : 
    2594             : static void
    2595           2 : raid_bdev_process_finish_quiesced(void *ctx, int status)
    2596             : {
    2597           2 :         struct raid_bdev_process *process = ctx;
    2598           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2599             : 
    2600           2 :         if (status != 0) {
    2601           0 :                 SPDK_ERRLOG("Failed to quiesce bdev: %s\n", spdk_strerror(-status));
    2602           0 :                 return;
    2603             :         }
    2604             : 
    2605           2 :         raid_bdev->process = NULL;
    2606           2 :         process->target->is_process_target = false;
    2607             : 
    2608           2 :         spdk_for_each_channel(process->raid_bdev, raid_bdev_channel_process_finish, process,
    2609             :                               __raid_bdev_process_finish);
    2610             : }
    2611             : 
    2612             : static void
    2613           2 : _raid_bdev_process_finish(void *ctx)
    2614             : {
    2615           2 :         struct raid_bdev_process *process = ctx;
    2616             :         int rc;
    2617             : 
    2618           2 :         rc = spdk_bdev_quiesce(&process->raid_bdev->bdev, &g_raid_if,
    2619             :                                raid_bdev_process_finish_quiesced, process);
    2620           2 :         if (rc != 0) {
    2621           0 :                 raid_bdev_process_finish_quiesced(ctx, rc);
    2622             :         }
    2623           2 : }
    2624             : 
    2625             : static void
    2626           2 : raid_bdev_process_do_finish(struct raid_bdev_process *process)
    2627             : {
    2628           2 :         spdk_thread_send_msg(spdk_thread_get_app_thread(), _raid_bdev_process_finish, process);
    2629           2 : }
    2630             : 
    2631             : static void raid_bdev_process_unlock_window_range(struct raid_bdev_process *process);
    2632             : static void raid_bdev_process_thread_run(struct raid_bdev_process *process);
    2633             : 
    2634             : static void
    2635           2 : raid_bdev_process_finish(struct raid_bdev_process *process, int status)
    2636             : {
    2637           2 :         assert(spdk_get_thread() == process->thread);
    2638             : 
    2639           2 :         if (process->status == 0) {
    2640           2 :                 process->status = status;
    2641             :         }
    2642             : 
    2643           2 :         if (process->state >= RAID_PROCESS_STATE_STOPPING) {
    2644           0 :                 return;
    2645             :         }
    2646             : 
    2647           2 :         assert(process->state == RAID_PROCESS_STATE_RUNNING);
    2648           2 :         process->state = RAID_PROCESS_STATE_STOPPING;
    2649             : 
    2650           2 :         if (process->window_range_locked) {
    2651           0 :                 raid_bdev_process_unlock_window_range(process);
    2652             :         } else {
    2653           2 :                 raid_bdev_process_thread_run(process);
    2654             :         }
    2655             : }
    2656             : 
    2657             : static void
    2658           2 : raid_bdev_process_window_range_unlocked(void *ctx, int status)
    2659             : {
    2660           2 :         struct raid_bdev_process *process = ctx;
    2661             : 
    2662           2 :         if (status != 0) {
    2663           0 :                 SPDK_ERRLOG("Failed to unlock LBA range: %s\n", spdk_strerror(-status));
    2664           0 :                 raid_bdev_process_finish(process, status);
    2665           0 :                 return;
    2666             :         }
    2667             : 
    2668           2 :         process->window_range_locked = false;
    2669           2 :         process->window_offset += process->window_size;
    2670             : 
    2671           2 :         raid_bdev_process_thread_run(process);
    2672             : }
    2673             : 
    2674             : static void
    2675           2 : raid_bdev_process_unlock_window_range(struct raid_bdev_process *process)
    2676             : {
    2677             :         int rc;
    2678             : 
    2679           2 :         assert(process->window_range_locked == true);
    2680             : 
    2681           2 :         rc = spdk_bdev_unquiesce_range(&process->raid_bdev->bdev, &g_raid_if,
    2682             :                                        process->window_offset, process->max_window_size,
    2683             :                                        raid_bdev_process_window_range_unlocked, process);
    2684           2 :         if (rc != 0) {
    2685           0 :                 raid_bdev_process_window_range_unlocked(process, rc);
    2686             :         }
    2687           2 : }
    2688             : 
    2689             : static void
    2690           2 : raid_bdev_process_channels_update_done(struct spdk_io_channel_iter *i, int status)
    2691             : {
    2692           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2693             : 
    2694           2 :         raid_bdev_process_unlock_window_range(process);
    2695           2 : }
    2696             : 
    2697             : static void
    2698           2 : raid_bdev_process_channel_update(struct spdk_io_channel_iter *i)
    2699             : {
    2700           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2701           2 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2702           2 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2703             : 
    2704           2 :         raid_ch->process.offset = process->window_offset + process->window_size;
    2705             : 
    2706           2 :         spdk_for_each_channel_continue(i, 0);
    2707           2 : }
    2708             : 
    2709             : void
    2710           2 : raid_bdev_process_request_complete(struct raid_bdev_process_request *process_req, int status)
    2711             : {
    2712           2 :         struct raid_bdev_process *process = process_req->process;
    2713             : 
    2714           2 :         TAILQ_INSERT_TAIL(&process->requests, process_req, link);
    2715             : 
    2716           2 :         assert(spdk_get_thread() == process->thread);
    2717           2 :         assert(process->window_remaining >= process_req->num_blocks);
    2718             : 
    2719           2 :         if (status != 0) {
    2720           0 :                 process->window_status = status;
    2721             :         }
    2722             : 
    2723           2 :         process->window_remaining -= process_req->num_blocks;
    2724           2 :         if (process->window_remaining == 0) {
    2725           2 :                 if (process->window_status != 0) {
    2726           0 :                         raid_bdev_process_finish(process, process->window_status);
    2727           0 :                         return;
    2728             :                 }
    2729             : 
    2730           2 :                 spdk_for_each_channel(process->raid_bdev, raid_bdev_process_channel_update, process,
    2731             :                                       raid_bdev_process_channels_update_done);
    2732             :         }
    2733             : }
    2734             : 
    2735             : static int
    2736           2 : raid_bdev_submit_process_request(struct raid_bdev_process *process, uint64_t offset_blocks,
    2737             :                                  uint32_t num_blocks)
    2738             : {
    2739           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2740             :         struct raid_bdev_process_request *process_req;
    2741             :         int ret;
    2742             : 
    2743           2 :         process_req = TAILQ_FIRST(&process->requests);
    2744           2 :         if (process_req == NULL) {
    2745           0 :                 assert(process->window_remaining > 0);
    2746           0 :                 return 0;
    2747             :         }
    2748             : 
    2749           2 :         process_req->target = process->target;
    2750           2 :         process_req->target_ch = process->raid_ch->process.target_ch;
    2751           2 :         process_req->offset_blocks = offset_blocks;
    2752           2 :         process_req->num_blocks = num_blocks;
    2753           2 :         process_req->iov.iov_len = num_blocks * raid_bdev->bdev.blocklen;
    2754             : 
    2755           2 :         ret = raid_bdev->module->submit_process_request(process_req, process->raid_ch);
    2756           2 :         if (ret <= 0) {
    2757           0 :                 if (ret < 0) {
    2758           0 :                         SPDK_ERRLOG("Failed to submit process request on %s: %s\n",
    2759             :                                     raid_bdev->bdev.name, spdk_strerror(-ret));
    2760           0 :                         process->window_status = ret;
    2761             :                 }
    2762           0 :                 return ret;
    2763             :         }
    2764             : 
    2765           2 :         process_req->num_blocks = ret;
    2766           2 :         TAILQ_REMOVE(&process->requests, process_req, link);
    2767             : 
    2768           2 :         return ret;
    2769             : }
    2770             : 
    2771             : static void
    2772           2 : _raid_bdev_process_thread_run(struct raid_bdev_process *process)
    2773             : {
    2774           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2775           2 :         uint64_t offset = process->window_offset;
    2776           2 :         const uint64_t offset_end = spdk_min(offset + process->max_window_size, raid_bdev->bdev.blockcnt);
    2777             :         int ret;
    2778             : 
    2779           4 :         while (offset < offset_end) {
    2780           2 :                 ret = raid_bdev_submit_process_request(process, offset, offset_end - offset);
    2781           2 :                 if (ret <= 0) {
    2782           0 :                         break;
    2783             :                 }
    2784             : 
    2785           2 :                 process->window_remaining += ret;
    2786           2 :                 offset += ret;
    2787             :         }
    2788             : 
    2789           2 :         if (process->window_remaining > 0) {
    2790           2 :                 process->window_size = process->window_remaining;
    2791             :         } else {
    2792           0 :                 raid_bdev_process_finish(process, process->window_status);
    2793             :         }
    2794           2 : }
    2795             : 
    2796             : static void
    2797           2 : raid_bdev_process_window_range_locked(void *ctx, int status)
    2798             : {
    2799           2 :         struct raid_bdev_process *process = ctx;
    2800             : 
    2801           2 :         if (status != 0) {
    2802           0 :                 SPDK_ERRLOG("Failed to lock LBA range: %s\n", spdk_strerror(-status));
    2803           0 :                 raid_bdev_process_finish(process, status);
    2804           0 :                 return;
    2805             :         }
    2806             : 
    2807           2 :         process->window_range_locked = true;
    2808             : 
    2809           2 :         if (process->state == RAID_PROCESS_STATE_STOPPING) {
    2810           0 :                 raid_bdev_process_unlock_window_range(process);
    2811           0 :                 return;
    2812             :         }
    2813             : 
    2814           2 :         _raid_bdev_process_thread_run(process);
    2815             : }
    2816             : 
    2817             : static bool
    2818          12 : raid_bdev_process_consume_token(struct raid_bdev_process *process)
    2819             : {
    2820          12 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2821          12 :         uint64_t now = spdk_get_ticks();
    2822             : 
    2823          12 :         process->qos.bytes_available = spdk_min(process->qos.bytes_max,
    2824             :                                                 process->qos.bytes_available +
    2825             :                                                 (now - process->qos.last_tsc) * process->qos.bytes_per_tsc);
    2826          12 :         process->qos.last_tsc = now;
    2827          12 :         if (process->qos.bytes_available > 0.0) {
    2828           1 :                 process->qos.bytes_available -= process->window_size * raid_bdev->bdev.blocklen;
    2829           1 :                 return true;
    2830             :         }
    2831          11 :         return false;
    2832             : }
    2833             : 
    2834             : static bool
    2835          13 : raid_bdev_process_lock_window_range(struct raid_bdev_process *process)
    2836             : {
    2837          13 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2838             :         int rc;
    2839             : 
    2840          13 :         assert(process->window_range_locked == false);
    2841             : 
    2842          13 :         if (process->qos.enable_qos) {
    2843          12 :                 if (raid_bdev_process_consume_token(process)) {
    2844           1 :                         spdk_poller_pause(process->qos.process_continue_poller);
    2845             :                 } else {
    2846          11 :                         spdk_poller_resume(process->qos.process_continue_poller);
    2847          11 :                         return false;
    2848             :                 }
    2849             :         }
    2850             : 
    2851           2 :         rc = spdk_bdev_quiesce_range(&raid_bdev->bdev, &g_raid_if,
    2852             :                                      process->window_offset, process->max_window_size,
    2853             :                                      raid_bdev_process_window_range_locked, process);
    2854           2 :         if (rc != 0) {
    2855           0 :                 raid_bdev_process_window_range_locked(process, rc);
    2856             :         }
    2857           2 :         return true;
    2858             : }
    2859             : 
    2860             : static int
    2861          11 : raid_bdev_process_continue_poll(void *arg)
    2862             : {
    2863          11 :         struct raid_bdev_process *process = arg;
    2864             : 
    2865          11 :         if (raid_bdev_process_lock_window_range(process)) {
    2866           1 :                 return SPDK_POLLER_BUSY;
    2867             :         }
    2868          10 :         return SPDK_POLLER_IDLE;
    2869             : }
    2870             : 
    2871             : static void
    2872           6 : raid_bdev_process_thread_run(struct raid_bdev_process *process)
    2873             : {
    2874           6 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2875             : 
    2876           6 :         assert(spdk_get_thread() == process->thread);
    2877           6 :         assert(process->window_remaining == 0);
    2878           6 :         assert(process->window_range_locked == false);
    2879             : 
    2880           6 :         if (process->state == RAID_PROCESS_STATE_STOPPING) {
    2881           2 :                 raid_bdev_process_do_finish(process);
    2882           2 :                 return;
    2883             :         }
    2884             : 
    2885           4 :         if (process->window_offset == raid_bdev->bdev.blockcnt) {
    2886           2 :                 SPDK_DEBUGLOG(bdev_raid, "process completed on %s\n", raid_bdev->bdev.name);
    2887           2 :                 raid_bdev_process_finish(process, 0);
    2888           2 :                 return;
    2889             :         }
    2890             : 
    2891           2 :         process->max_window_size = spdk_min(raid_bdev->bdev.blockcnt - process->window_offset,
    2892             :                                             process->max_window_size);
    2893           2 :         raid_bdev_process_lock_window_range(process);
    2894             : }
    2895             : 
    2896             : static void
    2897           2 : raid_bdev_process_thread_init(void *ctx)
    2898             : {
    2899           2 :         struct raid_bdev_process *process = ctx;
    2900           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2901             :         struct spdk_io_channel *ch;
    2902             : 
    2903           2 :         process->thread = spdk_get_thread();
    2904             : 
    2905           2 :         ch = spdk_get_io_channel(raid_bdev);
    2906           2 :         if (ch == NULL) {
    2907           0 :                 process->status = -ENOMEM;
    2908           0 :                 raid_bdev_process_do_finish(process);
    2909           0 :                 return;
    2910             :         }
    2911             : 
    2912           2 :         process->raid_ch = spdk_io_channel_get_ctx(ch);
    2913           2 :         process->state = RAID_PROCESS_STATE_RUNNING;
    2914             : 
    2915           2 :         if (process->qos.enable_qos) {
    2916           1 :                 process->qos.process_continue_poller = SPDK_POLLER_REGISTER(raid_bdev_process_continue_poll,
    2917             :                                                        process, 0);
    2918           1 :                 spdk_poller_pause(process->qos.process_continue_poller);
    2919             :         }
    2920             : 
    2921           2 :         SPDK_NOTICELOG("Started %s on raid bdev %s\n",
    2922             :                        raid_bdev_process_to_str(process->type), raid_bdev->bdev.name);
    2923             : 
    2924           2 :         raid_bdev_process_thread_run(process);
    2925             : }
    2926             : 
    2927             : static void
    2928           0 : raid_bdev_channels_abort_start_process_done(struct spdk_io_channel_iter *i, int status)
    2929             : {
    2930           0 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2931             : 
    2932           0 :         _raid_bdev_remove_base_bdev(process->target, NULL, NULL);
    2933           0 :         raid_bdev_process_free(process);
    2934             : 
    2935             :         /* TODO: update sb */
    2936           0 : }
    2937             : 
    2938             : static void
    2939           0 : raid_bdev_channel_abort_start_process(struct spdk_io_channel_iter *i)
    2940             : {
    2941           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2942           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2943             : 
    2944           0 :         raid_bdev_ch_process_cleanup(raid_ch);
    2945             : 
    2946           0 :         spdk_for_each_channel_continue(i, 0);
    2947           0 : }
    2948             : 
    2949             : static void
    2950           2 : raid_bdev_channels_start_process_done(struct spdk_io_channel_iter *i, int status)
    2951             : {
    2952           2 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2953           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    2954             :         struct spdk_thread *thread;
    2955           2 :         char thread_name[RAID_BDEV_SB_NAME_SIZE + 16];
    2956             : 
    2957           2 :         if (status == 0 &&
    2958           2 :             (process->target->remove_scheduled || !process->target->is_configured ||
    2959           2 :              raid_bdev->num_base_bdevs_operational <= raid_bdev->min_base_bdevs_operational)) {
    2960             :                 /* a base bdev was removed before we got here */
    2961           0 :                 status = -ENODEV;
    2962             :         }
    2963             : 
    2964           2 :         if (status != 0) {
    2965           0 :                 SPDK_ERRLOG("Failed to start %s on %s: %s\n",
    2966             :                             raid_bdev_process_to_str(process->type), raid_bdev->bdev.name,
    2967             :                             spdk_strerror(-status));
    2968           0 :                 goto err;
    2969             :         }
    2970             : 
    2971           2 :         snprintf(thread_name, sizeof(thread_name), "%s_%s",
    2972             :                  raid_bdev->bdev.name, raid_bdev_process_to_str(process->type));
    2973             : 
    2974           2 :         thread = spdk_thread_create(thread_name, NULL);
    2975           2 :         if (thread == NULL) {
    2976           0 :                 SPDK_ERRLOG("Failed to create %s thread for %s\n",
    2977             :                             raid_bdev_process_to_str(process->type), raid_bdev->bdev.name);
    2978           0 :                 goto err;
    2979             :         }
    2980             : 
    2981           2 :         raid_bdev->process = process;
    2982             : 
    2983           2 :         spdk_thread_send_msg(thread, raid_bdev_process_thread_init, process);
    2984             : 
    2985           2 :         return;
    2986           0 : err:
    2987           0 :         spdk_for_each_channel(process->raid_bdev, raid_bdev_channel_abort_start_process, process,
    2988             :                               raid_bdev_channels_abort_start_process_done);
    2989             : }
    2990             : 
    2991             : static void
    2992           0 : raid_bdev_channel_start_process(struct spdk_io_channel_iter *i)
    2993             : {
    2994           0 :         struct raid_bdev_process *process = spdk_io_channel_iter_get_ctx(i);
    2995           0 :         struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
    2996           0 :         struct raid_bdev_io_channel *raid_ch = spdk_io_channel_get_ctx(ch);
    2997             :         int rc;
    2998             : 
    2999           0 :         rc = raid_bdev_ch_process_setup(raid_ch, process);
    3000             : 
    3001           0 :         spdk_for_each_channel_continue(i, rc);
    3002           0 : }
    3003             : 
    3004             : static void
    3005           2 : raid_bdev_process_start(struct raid_bdev_process *process)
    3006             : {
    3007           2 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    3008             : 
    3009           2 :         assert(raid_bdev->module->submit_process_request != NULL);
    3010             : 
    3011           2 :         spdk_for_each_channel(raid_bdev, raid_bdev_channel_start_process, process,
    3012             :                               raid_bdev_channels_start_process_done);
    3013           2 : }
    3014             : 
    3015             : static void
    3016          32 : raid_bdev_process_request_free(struct raid_bdev_process_request *process_req)
    3017             : {
    3018          32 :         spdk_dma_free(process_req->iov.iov_base);
    3019          32 :         spdk_dma_free(process_req->md_buf);
    3020          32 :         free(process_req);
    3021          32 : }
    3022             : 
    3023             : static struct raid_bdev_process_request *
    3024          32 : raid_bdev_process_alloc_request(struct raid_bdev_process *process)
    3025             : {
    3026          32 :         struct raid_bdev *raid_bdev = process->raid_bdev;
    3027             :         struct raid_bdev_process_request *process_req;
    3028             : 
    3029          32 :         process_req = calloc(1, sizeof(*process_req));
    3030          32 :         if (process_req == NULL) {
    3031           0 :                 return NULL;
    3032             :         }
    3033             : 
    3034          32 :         process_req->process = process;
    3035          32 :         process_req->iov.iov_len = process->max_window_size * raid_bdev->bdev.blocklen;
    3036          32 :         process_req->iov.iov_base = spdk_dma_malloc(process_req->iov.iov_len, 4096, 0);
    3037          32 :         if (process_req->iov.iov_base == NULL) {
    3038           0 :                 free(process_req);
    3039           0 :                 return NULL;
    3040             :         }
    3041          32 :         if (spdk_bdev_is_md_separate(&raid_bdev->bdev)) {
    3042          32 :                 process_req->md_buf = spdk_dma_malloc(process->max_window_size * raid_bdev->bdev.md_len, 4096, 0);
    3043          32 :                 if (process_req->md_buf == NULL) {
    3044           0 :                         raid_bdev_process_request_free(process_req);
    3045           0 :                         return NULL;
    3046             :                 }
    3047             :         }
    3048             : 
    3049          32 :         return process_req;
    3050             : }
    3051             : 
    3052             : static void
    3053           2 : raid_bdev_process_free(struct raid_bdev_process *process)
    3054             : {
    3055             :         struct raid_bdev_process_request *process_req;
    3056             : 
    3057          34 :         while ((process_req = TAILQ_FIRST(&process->requests)) != NULL) {
    3058          32 :                 TAILQ_REMOVE(&process->requests, process_req, link);
    3059          32 :                 raid_bdev_process_request_free(process_req);
    3060             :         }
    3061             : 
    3062           2 :         free(process);
    3063           2 : }
    3064             : 
    3065             : static struct raid_bdev_process *
    3066           2 : raid_bdev_process_alloc(struct raid_bdev *raid_bdev, enum raid_process_type type,
    3067             :                         struct raid_base_bdev_info *target)
    3068             : {
    3069             :         struct raid_bdev_process *process;
    3070             :         struct raid_bdev_process_request *process_req;
    3071             :         int i;
    3072             : 
    3073           2 :         process = calloc(1, sizeof(*process));
    3074           2 :         if (process == NULL) {
    3075           0 :                 return NULL;
    3076             :         }
    3077             : 
    3078           2 :         process->raid_bdev = raid_bdev;
    3079           2 :         process->type = type;
    3080           2 :         process->target = target;
    3081           2 :         process->max_window_size = spdk_max(spdk_divide_round_up(g_opts.process_window_size_kb * 1024UL,
    3082             :                                             spdk_bdev_get_data_block_size(&raid_bdev->bdev)),
    3083             :                                             raid_bdev->bdev.write_unit_size);
    3084           2 :         TAILQ_INIT(&process->requests);
    3085           2 :         TAILQ_INIT(&process->finish_actions);
    3086             : 
    3087           2 :         if (g_opts.process_max_bandwidth_mb_sec != 0) {
    3088           1 :                 process->qos.enable_qos = true;
    3089           1 :                 process->qos.last_tsc = spdk_get_ticks();
    3090           2 :                 process->qos.bytes_per_tsc = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 /
    3091           1 :                                              spdk_get_ticks_hz();
    3092           1 :                 process->qos.bytes_max = g_opts.process_max_bandwidth_mb_sec * 1024 * 1024.0 / SPDK_SEC_TO_MSEC;
    3093           1 :                 process->qos.bytes_available = 0.0;
    3094             :         }
    3095             : 
    3096          34 :         for (i = 0; i < RAID_BDEV_PROCESS_MAX_QD; i++) {
    3097          32 :                 process_req = raid_bdev_process_alloc_request(process);
    3098          32 :                 if (process_req == NULL) {
    3099           0 :                         raid_bdev_process_free(process);
    3100           0 :                         return NULL;
    3101             :                 }
    3102             : 
    3103          32 :                 TAILQ_INSERT_TAIL(&process->requests, process_req, link);
    3104             :         }
    3105             : 
    3106           2 :         return process;
    3107             : }
    3108             : 
    3109             : static int
    3110           2 : raid_bdev_start_rebuild(struct raid_base_bdev_info *target)
    3111             : {
    3112             :         struct raid_bdev_process *process;
    3113             : 
    3114           2 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3115             : 
    3116           2 :         process = raid_bdev_process_alloc(target->raid_bdev, RAID_PROCESS_REBUILD, target);
    3117           2 :         if (process == NULL) {
    3118           0 :                 return -ENOMEM;
    3119             :         }
    3120             : 
    3121           2 :         raid_bdev_process_start(process);
    3122             : 
    3123           2 :         return 0;
    3124             : }
    3125             : 
    3126             : static void raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info);
    3127             : 
    3128             : static void
    3129           0 : _raid_bdev_configure_base_bdev_cont(struct spdk_io_channel_iter *i, int status)
    3130             : {
    3131           0 :         struct raid_base_bdev_info *base_info = spdk_io_channel_iter_get_ctx(i);
    3132             : 
    3133           0 :         raid_bdev_configure_base_bdev_cont(base_info);
    3134           0 : }
    3135             : 
    3136             : static void
    3137           0 : raid_bdev_ch_sync(struct spdk_io_channel_iter *i)
    3138             : {
    3139           0 :         spdk_for_each_channel_continue(i, 0);
    3140           0 : }
    3141             : 
    3142             : static void
    3143         510 : raid_bdev_configure_base_bdev_cont(struct raid_base_bdev_info *base_info)
    3144             : {
    3145         510 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    3146             :         raid_base_bdev_cb configure_cb;
    3147             :         int rc;
    3148             : 
    3149         510 :         if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational &&
    3150           0 :             base_info->is_process_target == false) {
    3151             :                 /* TODO: defer if rebuild in progress on another base bdev */
    3152           0 :                 assert(raid_bdev->process == NULL);
    3153           0 :                 assert(raid_bdev->state == RAID_BDEV_STATE_ONLINE);
    3154           0 :                 base_info->is_process_target = true;
    3155             :                 /* To assure is_process_target is set before is_configured when checked in raid_bdev_create_cb() */
    3156           0 :                 spdk_for_each_channel(raid_bdev, raid_bdev_ch_sync, base_info, _raid_bdev_configure_base_bdev_cont);
    3157           0 :                 return;
    3158             :         }
    3159             : 
    3160         510 :         base_info->is_configured = true;
    3161             : 
    3162         510 :         raid_bdev->num_base_bdevs_discovered++;
    3163         510 :         assert(raid_bdev->num_base_bdevs_discovered <= raid_bdev->num_base_bdevs);
    3164         510 :         assert(raid_bdev->num_base_bdevs_operational <= raid_bdev->num_base_bdevs);
    3165         510 :         assert(raid_bdev->num_base_bdevs_operational >= raid_bdev->min_base_bdevs_operational);
    3166             : 
    3167         510 :         configure_cb = base_info->configure_cb;
    3168         510 :         base_info->configure_cb = NULL;
    3169             :         /*
    3170             :          * Configure the raid bdev when the number of discovered base bdevs reaches the number
    3171             :          * of base bdevs we know to be operational members of the array. Usually this is equal
    3172             :          * to the total number of base bdevs (num_base_bdevs) but can be less - when the array is
    3173             :          * degraded.
    3174             :          */
    3175         510 :         if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs_operational) {
    3176          14 :                 rc = raid_bdev_configure(raid_bdev, configure_cb, base_info->configure_cb_ctx);
    3177          14 :                 if (rc != 0) {
    3178           0 :                         SPDK_ERRLOG("Failed to configure raid bdev: %s\n", spdk_strerror(-rc));
    3179             :                 } else {
    3180          14 :                         configure_cb = NULL;
    3181             :                 }
    3182         496 :         } else if (base_info->is_process_target) {
    3183           0 :                 raid_bdev->num_base_bdevs_operational++;
    3184           0 :                 rc = raid_bdev_start_rebuild(base_info);
    3185           0 :                 if (rc != 0) {
    3186           0 :                         SPDK_ERRLOG("Failed to start rebuild: %s\n", spdk_strerror(-rc));
    3187           0 :                         _raid_bdev_remove_base_bdev(base_info, NULL, NULL);
    3188             :                 }
    3189             :         } else {
    3190         496 :                 rc = 0;
    3191             :         }
    3192             : 
    3193         510 :         if (configure_cb != NULL) {
    3194         496 :                 configure_cb(base_info->configure_cb_ctx, rc);
    3195             :         }
    3196             : }
    3197             : 
    3198             : static void raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3199             :                                  raid_base_bdev_cb cb_fn, void *cb_ctx);
    3200             : 
    3201             : static void
    3202         510 : raid_bdev_configure_base_bdev_check_sb_cb(const struct raid_bdev_superblock *sb, int status,
    3203             :                 void *ctx)
    3204             : {
    3205         510 :         struct raid_base_bdev_info *base_info = ctx;
    3206         510 :         raid_base_bdev_cb configure_cb = base_info->configure_cb;
    3207             : 
    3208         510 :         switch (status) {
    3209           0 :         case 0:
    3210             :                 /* valid superblock found */
    3211           0 :                 base_info->configure_cb = NULL;
    3212           0 :                 if (spdk_uuid_compare(&base_info->raid_bdev->bdev.uuid, &sb->uuid) == 0) {
    3213           0 :                         struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(base_info->desc);
    3214             : 
    3215           0 :                         raid_bdev_free_base_bdev_resource(base_info);
    3216           0 :                         raid_bdev_examine_sb(sb, bdev, configure_cb, base_info->configure_cb_ctx);
    3217           0 :                         return;
    3218             :                 }
    3219           0 :                 SPDK_ERRLOG("Superblock of a different raid bdev found on bdev %s\n", base_info->name);
    3220           0 :                 status = -EEXIST;
    3221           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    3222           0 :                 break;
    3223         510 :         case -EINVAL:
    3224             :                 /* no valid superblock */
    3225         510 :                 raid_bdev_configure_base_bdev_cont(base_info);
    3226         510 :                 return;
    3227           0 :         default:
    3228           0 :                 SPDK_ERRLOG("Failed to examine bdev %s: %s\n",
    3229             :                             base_info->name, spdk_strerror(-status));
    3230           0 :                 break;
    3231             :         }
    3232             : 
    3233           0 :         if (configure_cb != NULL) {
    3234           0 :                 base_info->configure_cb = NULL;
    3235           0 :                 configure_cb(base_info->configure_cb_ctx, status);
    3236             :         }
    3237             : }
    3238             : 
    3239             : static int
    3240         513 : raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info, bool existing,
    3241             :                               raid_base_bdev_cb cb_fn, void *cb_ctx)
    3242             : {
    3243         513 :         struct raid_bdev *raid_bdev = base_info->raid_bdev;
    3244         513 :         struct spdk_bdev_desc *desc;
    3245             :         struct spdk_bdev *bdev;
    3246             :         const struct spdk_uuid *bdev_uuid;
    3247             :         int rc;
    3248             : 
    3249         513 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3250         513 :         assert(base_info->desc == NULL);
    3251             : 
    3252             :         /*
    3253             :          * Base bdev can be added by name or uuid. Here we assure both properties are set and valid
    3254             :          * before claiming the bdev.
    3255             :          */
    3256             : 
    3257         513 :         if (!spdk_uuid_is_null(&base_info->uuid)) {
    3258           0 :                 char uuid_str[SPDK_UUID_STRING_LEN];
    3259             :                 const char *bdev_name;
    3260             : 
    3261           0 :                 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &base_info->uuid);
    3262             : 
    3263             :                 /* UUID of a bdev is registered as its alias */
    3264           0 :                 bdev = spdk_bdev_get_by_name(uuid_str);
    3265           0 :                 if (bdev == NULL) {
    3266           0 :                         return -ENODEV;
    3267             :                 }
    3268             : 
    3269           0 :                 bdev_name = spdk_bdev_get_name(bdev);
    3270             : 
    3271           0 :                 if (base_info->name == NULL) {
    3272           0 :                         assert(existing == true);
    3273           0 :                         base_info->name = strdup(bdev_name);
    3274           0 :                         if (base_info->name == NULL) {
    3275           0 :                                 return -ENOMEM;
    3276             :                         }
    3277           0 :                 } else if (strcmp(base_info->name, bdev_name) != 0) {
    3278           0 :                         SPDK_ERRLOG("Name mismatch for base bdev '%s' - expected '%s'\n",
    3279             :                                     bdev_name, base_info->name);
    3280           0 :                         return -EINVAL;
    3281             :                 }
    3282             :         }
    3283             : 
    3284         513 :         assert(base_info->name != NULL);
    3285             : 
    3286         513 :         rc = spdk_bdev_open_ext(base_info->name, true, raid_bdev_event_base_bdev, NULL, &desc);
    3287         513 :         if (rc != 0) {
    3288           1 :                 if (rc != -ENODEV) {
    3289           0 :                         SPDK_ERRLOG("Unable to create desc on bdev '%s'\n", base_info->name);
    3290             :                 }
    3291           1 :                 return rc;
    3292             :         }
    3293             : 
    3294         512 :         bdev = spdk_bdev_desc_get_bdev(desc);
    3295         512 :         bdev_uuid = spdk_bdev_get_uuid(bdev);
    3296             : 
    3297         512 :         if (spdk_uuid_is_null(&base_info->uuid)) {
    3298         512 :                 spdk_uuid_copy(&base_info->uuid, bdev_uuid);
    3299           0 :         } else if (spdk_uuid_compare(&base_info->uuid, bdev_uuid) != 0) {
    3300           0 :                 SPDK_ERRLOG("UUID mismatch for base bdev '%s'\n", base_info->name);
    3301           0 :                 spdk_bdev_close(desc);
    3302           0 :                 return -EINVAL;
    3303             :         }
    3304             : 
    3305         512 :         rc = spdk_bdev_module_claim_bdev(bdev, NULL, &g_raid_if);
    3306         512 :         if (rc != 0) {
    3307           2 :                 SPDK_ERRLOG("Unable to claim this bdev as it is already claimed\n");
    3308           2 :                 spdk_bdev_close(desc);
    3309           2 :                 return rc;
    3310             :         }
    3311             : 
    3312         510 :         SPDK_DEBUGLOG(bdev_raid, "bdev %s is claimed\n", bdev->name);
    3313             : 
    3314         510 :         base_info->app_thread_ch = spdk_bdev_get_io_channel(desc);
    3315         510 :         if (base_info->app_thread_ch == NULL) {
    3316           0 :                 SPDK_ERRLOG("Failed to get io channel\n");
    3317           0 :                 spdk_bdev_module_release_bdev(bdev);
    3318           0 :                 spdk_bdev_close(desc);
    3319           0 :                 return -ENOMEM;
    3320             :         }
    3321             : 
    3322         510 :         base_info->desc = desc;
    3323         510 :         base_info->blockcnt = bdev->blockcnt;
    3324             : 
    3325         510 :         if (raid_bdev->superblock_enabled) {
    3326             :                 uint64_t data_offset;
    3327             : 
    3328          32 :                 if (base_info->data_offset == 0) {
    3329          32 :                         assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % spdk_bdev_get_data_block_size(bdev)) == 0);
    3330          32 :                         data_offset = RAID_BDEV_MIN_DATA_OFFSET_SIZE / spdk_bdev_get_data_block_size(bdev);
    3331             :                 } else {
    3332           0 :                         data_offset = base_info->data_offset;
    3333             :                 }
    3334             : 
    3335          32 :                 if (bdev->optimal_io_boundary != 0) {
    3336           0 :                         data_offset = spdk_divide_round_up(data_offset,
    3337           0 :                                                            bdev->optimal_io_boundary) * bdev->optimal_io_boundary;
    3338           0 :                         if (base_info->data_offset != 0 && base_info->data_offset != data_offset) {
    3339           0 :                                 SPDK_WARNLOG("Data offset %lu on bdev '%s' is different than optimal value %lu\n",
    3340             :                                              base_info->data_offset, base_info->name, data_offset);
    3341           0 :                                 data_offset = base_info->data_offset;
    3342             :                         }
    3343             :                 }
    3344             : 
    3345          32 :                 base_info->data_offset = data_offset;
    3346             :         }
    3347             : 
    3348         510 :         if (base_info->data_offset >= bdev->blockcnt) {
    3349           0 :                 SPDK_ERRLOG("Data offset %lu exceeds base bdev capacity %lu on bdev '%s'\n",
    3350             :                             base_info->data_offset, bdev->blockcnt, base_info->name);
    3351           0 :                 rc = -EINVAL;
    3352           0 :                 goto out;
    3353             :         }
    3354             : 
    3355         510 :         if (base_info->data_size == 0) {
    3356         510 :                 base_info->data_size = bdev->blockcnt - base_info->data_offset;
    3357           0 :         } else if (base_info->data_offset + base_info->data_size > bdev->blockcnt) {
    3358           0 :                 SPDK_ERRLOG("Data offset and size exceeds base bdev capacity %lu on bdev '%s'\n",
    3359             :                             bdev->blockcnt, base_info->name);
    3360           0 :                 rc = -EINVAL;
    3361           0 :                 goto out;
    3362             :         }
    3363             : 
    3364         510 :         if (!raid_bdev->module->dif_supported && spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
    3365           0 :                 SPDK_ERRLOG("Base bdev '%s' has DIF or DIX enabled - unsupported RAID configuration\n",
    3366             :                             bdev->name);
    3367           0 :                 rc = -EINVAL;
    3368           0 :                 goto out;
    3369             :         }
    3370             : 
    3371             :         /*
    3372             :          * Set the raid bdev properties if this is the first base bdev configured,
    3373             :          * otherwise - verify. Assumption is that all the base bdevs for any raid bdev should
    3374             :          * have the same blocklen and metadata format.
    3375             :          */
    3376         510 :         if (raid_bdev->bdev.blocklen == 0) {
    3377          16 :                 raid_bdev->bdev.blocklen = bdev->blocklen;
    3378          16 :                 raid_bdev->bdev.md_len = spdk_bdev_get_md_size(bdev);
    3379          16 :                 raid_bdev->bdev.md_interleave = spdk_bdev_is_md_interleaved(bdev);
    3380          16 :                 raid_bdev->bdev.dif_type = spdk_bdev_get_dif_type(bdev);
    3381          16 :                 raid_bdev->bdev.dif_check_flags = bdev->dif_check_flags;
    3382          16 :                 raid_bdev->bdev.dif_is_head_of_md = spdk_bdev_is_dif_head_of_md(bdev);
    3383          16 :                 raid_bdev->bdev.dif_pi_format = bdev->dif_pi_format;
    3384             :         } else {
    3385         494 :                 if (raid_bdev->bdev.blocklen != bdev->blocklen) {
    3386           0 :                         SPDK_ERRLOG("Raid bdev '%s' blocklen %u differs from base bdev '%s' blocklen %u\n",
    3387             :                                     raid_bdev->bdev.name, raid_bdev->bdev.blocklen, bdev->name, bdev->blocklen);
    3388           0 :                         rc = -EINVAL;
    3389           0 :                         goto out;
    3390             :                 }
    3391             : 
    3392         988 :                 if (raid_bdev->bdev.md_len != spdk_bdev_get_md_size(bdev) ||
    3393         988 :                     raid_bdev->bdev.md_interleave != spdk_bdev_is_md_interleaved(bdev) ||
    3394         494 :                     raid_bdev->bdev.dif_type != spdk_bdev_get_dif_type(bdev) ||
    3395         988 :                     raid_bdev->bdev.dif_check_flags != bdev->dif_check_flags ||
    3396         494 :                     raid_bdev->bdev.dif_is_head_of_md != spdk_bdev_is_dif_head_of_md(bdev) ||
    3397         494 :                     raid_bdev->bdev.dif_pi_format != bdev->dif_pi_format) {
    3398           0 :                         SPDK_ERRLOG("Raid bdev '%s' has different metadata format than base bdev '%s'\n",
    3399             :                                     raid_bdev->bdev.name, bdev->name);
    3400           0 :                         rc = -EINVAL;
    3401           0 :                         goto out;
    3402             :                 }
    3403             :         }
    3404             : 
    3405         510 :         assert(base_info->configure_cb == NULL);
    3406         510 :         base_info->configure_cb = cb_fn;
    3407         510 :         base_info->configure_cb_ctx = cb_ctx;
    3408             : 
    3409         510 :         if (existing) {
    3410           0 :                 raid_bdev_configure_base_bdev_cont(base_info);
    3411             :         } else {
    3412             :                 /* check for existing superblock when using a new bdev */
    3413         510 :                 rc = raid_bdev_load_base_bdev_superblock(desc, base_info->app_thread_ch,
    3414             :                                 raid_bdev_configure_base_bdev_check_sb_cb, base_info);
    3415         510 :                 if (rc) {
    3416           0 :                         SPDK_ERRLOG("Failed to read bdev %s superblock: %s\n",
    3417             :                                     bdev->name, spdk_strerror(-rc));
    3418             :                 }
    3419             :         }
    3420         510 : out:
    3421         510 :         if (rc != 0) {
    3422           0 :                 base_info->configure_cb = NULL;
    3423           0 :                 raid_bdev_free_base_bdev_resource(base_info);
    3424             :         }
    3425         510 :         return rc;
    3426             : }
    3427             : 
    3428             : int
    3429         513 : raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, const char *name,
    3430             :                         raid_base_bdev_cb cb_fn, void *cb_ctx)
    3431             : {
    3432         513 :         struct raid_base_bdev_info *base_info = NULL, *iter;
    3433             :         int rc;
    3434             : 
    3435         513 :         assert(name != NULL);
    3436         513 :         assert(spdk_get_thread() == spdk_thread_get_app_thread());
    3437             : 
    3438         513 :         if (raid_bdev->process != NULL) {
    3439           0 :                 SPDK_ERRLOG("raid bdev '%s' is in process\n",
    3440             :                             raid_bdev->bdev.name);
    3441           0 :                 return -EPERM;
    3442             :         }
    3443             : 
    3444         513 :         if (raid_bdev->state == RAID_BDEV_STATE_CONFIGURING) {
    3445         513 :                 struct spdk_bdev *bdev = spdk_bdev_get_by_name(name);
    3446             : 
    3447         513 :                 if (bdev != NULL) {
    3448       16896 :                         RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3449       24863 :                                 if (iter->name == NULL &&
    3450        8479 :                                     spdk_uuid_compare(&bdev->uuid, &iter->uuid) == 0) {
    3451           0 :                                         base_info = iter;
    3452           0 :                                         break;
    3453             :                                 }
    3454             :                         }
    3455             :                 }
    3456             :         }
    3457             : 
    3458         513 :         if (base_info == NULL || raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3459        8449 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3460        8449 :                         if (iter->name == NULL && spdk_uuid_is_null(&iter->uuid)) {
    3461         513 :                                 base_info = iter;
    3462         513 :                                 break;
    3463             :                         }
    3464             :                 }
    3465             :         }
    3466             : 
    3467         513 :         if (base_info == NULL) {
    3468           0 :                 SPDK_ERRLOG("no empty slot found in raid bdev '%s' for new base bdev '%s'\n",
    3469             :                             raid_bdev->bdev.name, name);
    3470           0 :                 return -EINVAL;
    3471             :         }
    3472             : 
    3473         513 :         assert(base_info->is_configured == false);
    3474             : 
    3475         513 :         if (raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3476           0 :                 assert(base_info->data_size != 0);
    3477           0 :                 assert(base_info->desc == NULL);
    3478             :         }
    3479             : 
    3480         513 :         base_info->name = strdup(name);
    3481         513 :         if (base_info->name == NULL) {
    3482           0 :                 return -ENOMEM;
    3483             :         }
    3484             : 
    3485         513 :         rc = raid_bdev_configure_base_bdev(base_info, false, cb_fn, cb_ctx);
    3486         513 :         if (rc != 0 && (rc != -ENODEV || raid_bdev->state != RAID_BDEV_STATE_CONFIGURING)) {
    3487           2 :                 SPDK_ERRLOG("base bdev '%s' configure failed: %s\n", name, spdk_strerror(-rc));
    3488           2 :                 free(base_info->name);
    3489           2 :                 base_info->name = NULL;
    3490             :         }
    3491             : 
    3492         513 :         return rc;
    3493             : }
    3494             : 
    3495             : static int
    3496           0 : raid_bdev_create_from_sb(const struct raid_bdev_superblock *sb, struct raid_bdev **raid_bdev_out)
    3497             : {
    3498           0 :         struct raid_bdev *raid_bdev;
    3499             :         uint8_t i;
    3500             :         int rc;
    3501             : 
    3502           0 :         rc = _raid_bdev_create(sb->name, (sb->strip_size * sb->block_size) / 1024, sb->num_base_bdevs,
    3503           0 :                                sb->level, true, &sb->uuid, &raid_bdev);
    3504           0 :         if (rc != 0) {
    3505           0 :                 return rc;
    3506             :         }
    3507             : 
    3508           0 :         rc = raid_bdev_alloc_superblock(raid_bdev, sb->block_size);
    3509           0 :         if (rc != 0) {
    3510           0 :                 raid_bdev_free(raid_bdev);
    3511           0 :                 return rc;
    3512             :         }
    3513             : 
    3514           0 :         assert(sb->length <= RAID_BDEV_SB_MAX_LENGTH);
    3515           0 :         memcpy(raid_bdev->sb, sb, sb->length);
    3516             : 
    3517           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    3518           0 :                 const struct raid_bdev_sb_base_bdev *sb_base_bdev = &sb->base_bdevs[i];
    3519           0 :                 struct raid_base_bdev_info *base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    3520             : 
    3521           0 :                 if (sb_base_bdev->state == RAID_SB_BASE_BDEV_CONFIGURED) {
    3522           0 :                         spdk_uuid_copy(&base_info->uuid, &sb_base_bdev->uuid);
    3523           0 :                         raid_bdev->num_base_bdevs_operational++;
    3524             :                 }
    3525             : 
    3526           0 :                 base_info->data_offset = sb_base_bdev->data_offset;
    3527           0 :                 base_info->data_size = sb_base_bdev->data_size;
    3528             :         }
    3529             : 
    3530           0 :         *raid_bdev_out = raid_bdev;
    3531           0 :         return 0;
    3532             : }
    3533             : 
    3534             : static void
    3535           0 : raid_bdev_examine_no_sb(struct spdk_bdev *bdev)
    3536             : {
    3537             :         struct raid_bdev *raid_bdev;
    3538             :         struct raid_base_bdev_info *base_info;
    3539             : 
    3540           0 :         TAILQ_FOREACH(raid_bdev, &g_raid_bdev_list, global_link) {
    3541           0 :                 if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING || raid_bdev->sb != NULL) {
    3542           0 :                         continue;
    3543             :                 }
    3544           0 :                 RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
    3545           0 :                         if (base_info->desc == NULL &&
    3546           0 :                             ((base_info->name != NULL && strcmp(bdev->name, base_info->name) == 0) ||
    3547           0 :                              spdk_uuid_compare(&base_info->uuid, &bdev->uuid) == 0)) {
    3548           0 :                                 raid_bdev_configure_base_bdev(base_info, true, NULL, NULL);
    3549           0 :                                 break;
    3550             :                         }
    3551             :                 }
    3552             :         }
    3553           0 : }
    3554             : 
    3555             : struct raid_bdev_examine_others_ctx {
    3556             :         struct spdk_uuid raid_bdev_uuid;
    3557             :         uint8_t current_base_bdev_idx;
    3558             :         raid_base_bdev_cb cb_fn;
    3559             :         void *cb_ctx;
    3560             : };
    3561             : 
    3562             : static void
    3563           0 : raid_bdev_examine_others_done(void *_ctx, int status)
    3564             : {
    3565           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3566             : 
    3567           0 :         if (ctx->cb_fn != NULL) {
    3568           0 :                 ctx->cb_fn(ctx->cb_ctx, status);
    3569             :         }
    3570           0 :         free(ctx);
    3571           0 : }
    3572             : 
    3573             : typedef void (*raid_bdev_examine_load_sb_cb)(struct spdk_bdev *bdev,
    3574             :                 const struct raid_bdev_superblock *sb, int status, void *ctx);
    3575             : static int raid_bdev_examine_load_sb(const char *bdev_name, raid_bdev_examine_load_sb_cb cb,
    3576             :                                      void *cb_ctx);
    3577             : static void raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3578             :                                  raid_base_bdev_cb cb_fn, void *cb_ctx);
    3579             : static void raid_bdev_examine_others(void *_ctx, int status);
    3580             : 
    3581             : static void
    3582           0 : raid_bdev_examine_others_load_cb(struct spdk_bdev *bdev, const struct raid_bdev_superblock *sb,
    3583             :                                  int status, void *_ctx)
    3584             : {
    3585           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3586             : 
    3587           0 :         if (status != 0) {
    3588           0 :                 raid_bdev_examine_others_done(ctx, status);
    3589           0 :                 return;
    3590             :         }
    3591             : 
    3592           0 :         raid_bdev_examine_sb(sb, bdev, raid_bdev_examine_others, ctx);
    3593             : }
    3594             : 
    3595             : static void
    3596           0 : raid_bdev_examine_others(void *_ctx, int status)
    3597             : {
    3598           0 :         struct raid_bdev_examine_others_ctx *ctx = _ctx;
    3599             :         struct raid_bdev *raid_bdev;
    3600             :         struct raid_base_bdev_info *base_info;
    3601           0 :         char uuid_str[SPDK_UUID_STRING_LEN];
    3602             : 
    3603           0 :         if (status != 0 && status != -EEXIST) {
    3604           0 :                 goto out;
    3605             :         }
    3606             : 
    3607           0 :         raid_bdev = raid_bdev_find_by_uuid(&ctx->raid_bdev_uuid);
    3608           0 :         if (raid_bdev == NULL) {
    3609           0 :                 status = -ENODEV;
    3610           0 :                 goto out;
    3611             :         }
    3612             : 
    3613           0 :         for (base_info = &raid_bdev->base_bdev_info[ctx->current_base_bdev_idx];
    3614           0 :              base_info < &raid_bdev->base_bdev_info[raid_bdev->num_base_bdevs];
    3615           0 :              base_info++) {
    3616           0 :                 if (base_info->is_configured || spdk_uuid_is_null(&base_info->uuid)) {
    3617           0 :                         continue;
    3618             :                 }
    3619             : 
    3620           0 :                 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &base_info->uuid);
    3621             : 
    3622           0 :                 if (spdk_bdev_get_by_name(uuid_str) == NULL) {
    3623           0 :                         continue;
    3624             :                 }
    3625             : 
    3626           0 :                 ctx->current_base_bdev_idx = raid_bdev_base_bdev_slot(base_info);
    3627             : 
    3628           0 :                 status = raid_bdev_examine_load_sb(uuid_str, raid_bdev_examine_others_load_cb, ctx);
    3629           0 :                 if (status != 0) {
    3630           0 :                         continue;
    3631             :                 }
    3632           0 :                 return;
    3633             :         }
    3634           0 : out:
    3635           0 :         raid_bdev_examine_others_done(ctx, status);
    3636             : }
    3637             : 
    3638             : static void
    3639           0 : raid_bdev_examine_sb(const struct raid_bdev_superblock *sb, struct spdk_bdev *bdev,
    3640             :                      raid_base_bdev_cb cb_fn, void *cb_ctx)
    3641             : {
    3642           0 :         const struct raid_bdev_sb_base_bdev *sb_base_bdev = NULL;
    3643           0 :         struct raid_bdev *raid_bdev;
    3644             :         struct raid_base_bdev_info *iter, *base_info;
    3645             :         uint8_t i;
    3646             :         int rc;
    3647             : 
    3648           0 :         if (sb->block_size != spdk_bdev_get_data_block_size(bdev)) {
    3649           0 :                 SPDK_WARNLOG("Bdev %s block size (%u) does not match the value in superblock (%u)\n",
    3650             :                              bdev->name, sb->block_size, spdk_bdev_get_data_block_size(bdev));
    3651           0 :                 rc = -EINVAL;
    3652           0 :                 goto out;
    3653             :         }
    3654             : 
    3655           0 :         if (spdk_uuid_is_null(&sb->uuid)) {
    3656           0 :                 SPDK_WARNLOG("NULL raid bdev UUID in superblock on bdev %s\n", bdev->name);
    3657           0 :                 rc = -EINVAL;
    3658           0 :                 goto out;
    3659             :         }
    3660             : 
    3661           0 :         raid_bdev = raid_bdev_find_by_uuid(&sb->uuid);
    3662             : 
    3663           0 :         if (raid_bdev) {
    3664           0 :                 if (sb->seq_number > raid_bdev->sb->seq_number) {
    3665           0 :                         SPDK_DEBUGLOG(bdev_raid,
    3666             :                                       "raid superblock seq_number on bdev %s (%lu) greater than existing raid bdev %s (%lu)\n",
    3667             :                                       bdev->name, sb->seq_number, raid_bdev->bdev.name, raid_bdev->sb->seq_number);
    3668             : 
    3669           0 :                         if (raid_bdev->state != RAID_BDEV_STATE_CONFIGURING) {
    3670           0 :                                 SPDK_WARNLOG("Newer version of raid bdev %s superblock found on bdev %s but raid bdev is not in configuring state.\n",
    3671             :                                              raid_bdev->bdev.name, bdev->name);
    3672           0 :                                 rc = -EBUSY;
    3673           0 :                                 goto out;
    3674             :                         }
    3675             : 
    3676             :                         /* remove and then recreate the raid bdev using the newer superblock */
    3677           0 :                         raid_bdev_delete(raid_bdev, NULL, NULL);
    3678           0 :                         raid_bdev = NULL;
    3679           0 :                 } else if (sb->seq_number < raid_bdev->sb->seq_number) {
    3680           0 :                         SPDK_DEBUGLOG(bdev_raid,
    3681             :                                       "raid superblock seq_number on bdev %s (%lu) smaller than existing raid bdev %s (%lu)\n",
    3682             :                                       bdev->name, sb->seq_number, raid_bdev->bdev.name, raid_bdev->sb->seq_number);
    3683             :                         /* use the current raid bdev superblock */
    3684           0 :                         sb = raid_bdev->sb;
    3685             :                 }
    3686             :         }
    3687             : 
    3688           0 :         for (i = 0; i < sb->base_bdevs_size; i++) {
    3689           0 :                 sb_base_bdev = &sb->base_bdevs[i];
    3690             : 
    3691           0 :                 assert(spdk_uuid_is_null(&sb_base_bdev->uuid) == false);
    3692             : 
    3693           0 :                 if (spdk_uuid_compare(&sb_base_bdev->uuid, spdk_bdev_get_uuid(bdev)) == 0) {
    3694           0 :                         break;
    3695             :                 }
    3696             :         }
    3697             : 
    3698           0 :         if (i == sb->base_bdevs_size) {
    3699           0 :                 SPDK_DEBUGLOG(bdev_raid, "raid superblock does not contain this bdev's uuid\n");
    3700           0 :                 rc = -EINVAL;
    3701           0 :                 goto out;
    3702             :         }
    3703             : 
    3704           0 :         if (!raid_bdev) {
    3705             :                 struct raid_bdev_examine_others_ctx *ctx;
    3706             : 
    3707           0 :                 ctx = calloc(1, sizeof(*ctx));
    3708           0 :                 if (ctx == NULL) {
    3709           0 :                         rc = -ENOMEM;
    3710           0 :                         goto out;
    3711             :                 }
    3712             : 
    3713           0 :                 rc = raid_bdev_create_from_sb(sb, &raid_bdev);
    3714           0 :                 if (rc != 0) {
    3715           0 :                         SPDK_ERRLOG("Failed to create raid bdev %s: %s\n",
    3716             :                                     sb->name, spdk_strerror(-rc));
    3717           0 :                         free(ctx);
    3718           0 :                         goto out;
    3719             :                 }
    3720             : 
    3721             :                 /* after this base bdev is configured, examine other base bdevs that may be present */
    3722           0 :                 spdk_uuid_copy(&ctx->raid_bdev_uuid, &sb->uuid);
    3723           0 :                 ctx->cb_fn = cb_fn;
    3724           0 :                 ctx->cb_ctx = cb_ctx;
    3725             : 
    3726           0 :                 cb_fn = raid_bdev_examine_others;
    3727           0 :                 cb_ctx = ctx;
    3728             :         }
    3729             : 
    3730           0 :         if (raid_bdev->state == RAID_BDEV_STATE_ONLINE) {
    3731           0 :                 assert(sb_base_bdev->slot < raid_bdev->num_base_bdevs);
    3732           0 :                 base_info = &raid_bdev->base_bdev_info[sb_base_bdev->slot];
    3733           0 :                 assert(base_info->is_configured == false);
    3734           0 :                 assert(sb_base_bdev->state == RAID_SB_BASE_BDEV_MISSING ||
    3735             :                        sb_base_bdev->state == RAID_SB_BASE_BDEV_FAILED);
    3736           0 :                 assert(spdk_uuid_is_null(&base_info->uuid));
    3737           0 :                 spdk_uuid_copy(&base_info->uuid, &sb_base_bdev->uuid);
    3738           0 :                 SPDK_NOTICELOG("Re-adding bdev %s to raid bdev %s.\n", bdev->name, raid_bdev->bdev.name);
    3739           0 :                 rc = raid_bdev_configure_base_bdev(base_info, true, cb_fn, cb_ctx);
    3740           0 :                 if (rc != 0) {
    3741           0 :                         SPDK_ERRLOG("Failed to configure bdev %s as base bdev of raid %s: %s\n",
    3742             :                                     bdev->name, raid_bdev->bdev.name, spdk_strerror(-rc));
    3743             :                 }
    3744           0 :                 goto out;
    3745             :         }
    3746             : 
    3747           0 :         if (sb_base_bdev->state != RAID_SB_BASE_BDEV_CONFIGURED) {
    3748           0 :                 SPDK_NOTICELOG("Bdev %s is not an active member of raid bdev %s. Ignoring.\n",
    3749             :                                bdev->name, raid_bdev->bdev.name);
    3750           0 :                 rc = -EINVAL;
    3751           0 :                 goto out;
    3752             :         }
    3753             : 
    3754           0 :         base_info = NULL;
    3755           0 :         RAID_FOR_EACH_BASE_BDEV(raid_bdev, iter) {
    3756           0 :                 if (spdk_uuid_compare(&iter->uuid, spdk_bdev_get_uuid(bdev)) == 0) {
    3757           0 :                         base_info = iter;
    3758           0 :                         break;
    3759             :                 }
    3760             :         }
    3761             : 
    3762           0 :         if (base_info == NULL) {
    3763           0 :                 SPDK_ERRLOG("Bdev %s is not a member of raid bdev %s\n",
    3764             :                             bdev->name, raid_bdev->bdev.name);
    3765           0 :                 rc = -EINVAL;
    3766           0 :                 goto out;
    3767             :         }
    3768             : 
    3769           0 :         if (base_info->is_configured) {
    3770           0 :                 rc = -EEXIST;
    3771           0 :                 goto out;
    3772             :         }
    3773             : 
    3774           0 :         rc = raid_bdev_configure_base_bdev(base_info, true, cb_fn, cb_ctx);
    3775           0 :         if (rc != 0) {
    3776           0 :                 SPDK_ERRLOG("Failed to configure bdev %s as base bdev of raid %s: %s\n",
    3777             :                             bdev->name, raid_bdev->bdev.name, spdk_strerror(-rc));
    3778             :         }
    3779           0 : out:
    3780           0 :         if (rc != 0 && cb_fn != 0) {
    3781           0 :                 cb_fn(cb_ctx, rc);
    3782             :         }
    3783           0 : }
    3784             : 
    3785             : struct raid_bdev_examine_ctx {
    3786             :         struct spdk_bdev_desc *desc;
    3787             :         struct spdk_io_channel *ch;
    3788             :         raid_bdev_examine_load_sb_cb cb;
    3789             :         void *cb_ctx;
    3790             : };
    3791             : 
    3792             : static void
    3793           0 : raid_bdev_examine_ctx_free(struct raid_bdev_examine_ctx *ctx)
    3794             : {
    3795           0 :         if (!ctx) {
    3796           0 :                 return;
    3797             :         }
    3798             : 
    3799           0 :         if (ctx->ch) {
    3800           0 :                 spdk_put_io_channel(ctx->ch);
    3801             :         }
    3802             : 
    3803           0 :         if (ctx->desc) {
    3804           0 :                 spdk_bdev_close(ctx->desc);
    3805             :         }
    3806             : 
    3807           0 :         free(ctx);
    3808             : }
    3809             : 
    3810             : static void
    3811           0 : raid_bdev_examine_load_sb_done(const struct raid_bdev_superblock *sb, int status, void *_ctx)
    3812             : {
    3813           0 :         struct raid_bdev_examine_ctx *ctx = _ctx;
    3814           0 :         struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(ctx->desc);
    3815             : 
    3816           0 :         ctx->cb(bdev, sb, status, ctx->cb_ctx);
    3817             : 
    3818           0 :         raid_bdev_examine_ctx_free(ctx);
    3819           0 : }
    3820             : 
    3821             : static void
    3822           0 : raid_bdev_examine_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *event_ctx)
    3823             : {
    3824           0 : }
    3825             : 
    3826             : static int
    3827           0 : raid_bdev_examine_load_sb(const char *bdev_name, raid_bdev_examine_load_sb_cb cb, void *cb_ctx)
    3828             : {
    3829             :         struct raid_bdev_examine_ctx *ctx;
    3830             :         int rc;
    3831             : 
    3832           0 :         assert(cb != NULL);
    3833             : 
    3834           0 :         ctx = calloc(1, sizeof(*ctx));
    3835           0 :         if (!ctx) {
    3836           0 :                 return -ENOMEM;
    3837             :         }
    3838             : 
    3839           0 :         rc = spdk_bdev_open_ext(bdev_name, false, raid_bdev_examine_event_cb, NULL, &ctx->desc);
    3840           0 :         if (rc) {
    3841           0 :                 SPDK_ERRLOG("Failed to open bdev %s: %s\n", bdev_name, spdk_strerror(-rc));
    3842           0 :                 goto err;
    3843             :         }
    3844             : 
    3845           0 :         ctx->ch = spdk_bdev_get_io_channel(ctx->desc);
    3846           0 :         if (!ctx->ch) {
    3847           0 :                 SPDK_ERRLOG("Failed to get io channel for bdev %s\n", bdev_name);
    3848           0 :                 rc = -ENOMEM;
    3849           0 :                 goto err;
    3850             :         }
    3851             : 
    3852           0 :         ctx->cb = cb;
    3853           0 :         ctx->cb_ctx = cb_ctx;
    3854             : 
    3855           0 :         rc = raid_bdev_load_base_bdev_superblock(ctx->desc, ctx->ch, raid_bdev_examine_load_sb_done, ctx);
    3856           0 :         if (rc) {
    3857           0 :                 SPDK_ERRLOG("Failed to read bdev %s superblock: %s\n",
    3858             :                             bdev_name, spdk_strerror(-rc));
    3859           0 :                 goto err;
    3860             :         }
    3861             : 
    3862           0 :         return 0;
    3863           0 : err:
    3864           0 :         raid_bdev_examine_ctx_free(ctx);
    3865           0 :         return rc;
    3866             : }
    3867             : 
    3868             : static void
    3869           0 : raid_bdev_examine_cont(struct spdk_bdev *bdev, const struct raid_bdev_superblock *sb, int status,
    3870             :                        void *ctx)
    3871             : {
    3872           0 :         switch (status) {
    3873           0 :         case 0:
    3874             :                 /* valid superblock found */
    3875           0 :                 SPDK_DEBUGLOG(bdev_raid, "raid superblock found on bdev %s\n", bdev->name);
    3876           0 :                 raid_bdev_examine_sb(sb, bdev, NULL, NULL);
    3877           0 :                 break;
    3878           0 :         case -EINVAL:
    3879             :                 /* no valid superblock, check if it can be claimed anyway */
    3880           0 :                 raid_bdev_examine_no_sb(bdev);
    3881           0 :                 break;
    3882           0 :         default:
    3883           0 :                 SPDK_ERRLOG("Failed to examine bdev %s: %s\n",
    3884             :                             bdev->name, spdk_strerror(-status));
    3885           0 :                 break;
    3886             :         }
    3887             : 
    3888           0 :         spdk_bdev_module_examine_done(&g_raid_if);
    3889           0 : }
    3890             : 
    3891             : /*
    3892             :  * brief:
    3893             :  * raid_bdev_examine function is the examine function call by the below layers
    3894             :  * like bdev_nvme layer. This function will check if this base bdev can be
    3895             :  * claimed by this raid bdev or not.
    3896             :  * params:
    3897             :  * bdev - pointer to base bdev
    3898             :  * returns:
    3899             :  * none
    3900             :  */
    3901             : static void
    3902           0 : raid_bdev_examine(struct spdk_bdev *bdev)
    3903             : {
    3904             :         int rc;
    3905             : 
    3906           0 :         if (raid_bdev_find_base_info_by_bdev(bdev) != NULL) {
    3907           0 :                 goto done;
    3908             :         }
    3909             : 
    3910           0 :         if (spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
    3911           0 :                 raid_bdev_examine_no_sb(bdev);
    3912           0 :                 goto done;
    3913             :         }
    3914             : 
    3915           0 :         rc = raid_bdev_examine_load_sb(bdev->name, raid_bdev_examine_cont, NULL);
    3916           0 :         if (rc != 0) {
    3917           0 :                 SPDK_ERRLOG("Failed to examine bdev %s: %s\n",
    3918             :                             bdev->name, spdk_strerror(-rc));
    3919           0 :                 goto done;
    3920             :         }
    3921             : 
    3922           0 :         return;
    3923           0 : done:
    3924           0 :         spdk_bdev_module_examine_done(&g_raid_if);
    3925             : }
    3926             : 
    3927             : /* Log component for bdev raid bdev module */
    3928           1 : SPDK_LOG_REGISTER_COMPONENT(bdev_raid)

Generated by: LCOV version 1.15