Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2017 Intel Corporation.
3 : : * All rights reserved.
4 : : * Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #include "spdk/stdinc.h"
8 : :
9 : : #include "blobstore.h"
10 : : #include "request.h"
11 : :
12 : : #include "spdk/thread.h"
13 : : #include "spdk/queue.h"
14 : : #include "spdk/trace.h"
15 : :
16 : : #include "spdk_internal/trace_defs.h"
17 : : #include "spdk/log.h"
18 : :
19 : : void
20 : 7451608 : bs_call_cpl(struct spdk_bs_cpl *cpl, int bserrno)
21 : : {
22 [ + + + + : 7451608 : switch (cpl->type) {
+ + + + -
- - + ]
23 : 2510 : case SPDK_BS_CPL_TYPE_BS_BASIC:
24 [ # # # # : 3912 : cpl->u.bs_basic.cb_fn(cpl->u.bs_basic.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
25 : 701 : bserrno);
26 : 3211 : break;
27 : 7429 : case SPDK_BS_CPL_TYPE_BS_HANDLE:
28 [ + + + - : 9553 : cpl->u.bs_handle.cb_fn(cpl->u.bs_handle.cb_arg,
+ - + - -
+ + - + -
+ - + - +
- ]
29 [ + + # # : 1062 : bserrno == 0 ? cpl->u.bs_handle.bs : NULL,
# # # # #
# ]
30 : 1062 : bserrno);
31 : 8491 : break;
32 : 7354263 : case SPDK_BS_CPL_TYPE_BLOB_BASIC:
33 [ # # # # : 7451849 : cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
34 : 48793 : bserrno);
35 : 7403056 : break;
36 : 6739 : case SPDK_BS_CPL_TYPE_BLOBID:
37 [ + + # # : 10505 : cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
38 [ + + # # : 1883 : bserrno == 0 ? cpl->u.blobid.blobid : SPDK_BLOBID_INVALID,
# # # # #
# ]
39 : 1883 : bserrno);
40 : 8622 : break;
41 : 16214 : case SPDK_BS_CPL_TYPE_BLOB_HANDLE:
42 [ + + # # : 23226 : cpl->u.blob_handle.cb_fn(cpl->u.blob_handle.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
43 [ + + # # : 3506 : bserrno == 0 ? cpl->u.blob_handle.blob : NULL,
# # # # #
# ]
44 : 3506 : bserrno);
45 : 19720 : break;
46 : 0 : case SPDK_BS_CPL_TYPE_NESTED_SEQUENCE:
47 [ # # # # : 0 : cpl->u.nested_seq.cb_fn(cpl->u.nested_seq.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
48 [ # # # # : 0 : cpl->u.nested_seq.parent,
# # # # ]
49 : 0 : bserrno);
50 : 0 : break;
51 : 7442 : case SPDK_BS_CPL_TYPE_NONE:
52 : : /* this completion's callback is handled elsewhere */
53 : 8508 : break;
54 : : }
55 : 7451608 : }
56 : :
57 : : static void
58 : 7443100 : bs_request_set_complete(struct spdk_bs_request_set *set)
59 : : {
60 [ + - ]: 7443100 : struct spdk_bs_cpl cpl = set->cpl;
61 [ + - + - ]: 7443100 : int bserrno = set->bserrno;
62 : :
63 [ + + + + : 7443100 : spdk_trace_record(TRACE_BLOB_REQ_SET_COMPLETE, 0, 0, (uintptr_t)&set->cb_args,
+ - + - +
- + - + -
- + # # #
# # # # #
# # # # ]
64 : : (uintptr_t)set->cpl.u.blob_basic.cb_arg);
65 : :
66 [ + - + - : 7443100 : TAILQ_INSERT_TAIL(&set->channel->reqs, set, link);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - ]
67 : :
68 : 7443100 : bs_call_cpl(&cpl, bserrno);
69 : 7443100 : }
70 : :
71 : : static void
72 : 5726192 : bs_sequence_completion(struct spdk_io_channel *channel, void *cb_arg, int bserrno)
73 : : {
74 : 5726192 : struct spdk_bs_request_set *set = cb_arg;
75 : :
76 [ + - + - ]: 5726192 : set->bserrno = bserrno;
77 [ + - + - : 5726192 : set->u.sequence.cb_fn((spdk_bs_sequence_t *)set, set->u.sequence.cb_arg, bserrno);
+ - + - -
+ + - + -
+ - + - +
- ]
78 : 5726192 : }
79 : :
80 : : static inline spdk_bs_sequence_t *
81 : 5703267 : bs_sequence_start(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl,
82 : : struct spdk_io_channel *back_channel)
83 : : {
84 : : struct spdk_bs_channel *channel;
85 : : struct spdk_bs_request_set *set;
86 : :
87 : 5703267 : channel = spdk_io_channel_get_ctx(_channel);
88 [ + + # # ]: 5703267 : assert(channel != NULL);
89 [ + - + - : 5703267 : set = TAILQ_FIRST(&channel->reqs);
+ - ]
90 [ + + ]: 5703267 : if (!set) {
91 : 33243 : return NULL;
92 : : }
93 [ + + + - : 5670024 : TAILQ_REMOVE(&channel->reqs, set, link);
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - # # #
# # # # #
# # # # +
- + - + -
+ - + - +
- + - ]
94 : :
95 [ + + + + : 5670024 : spdk_trace_record(TRACE_BLOB_REQ_SET_START, 0, 0, (uintptr_t)&set->cb_args,
+ - + - +
- + - + -
- + # # #
# # # # #
# # ]
96 : : (uintptr_t)cpl->u.blob_basic.cb_arg);
97 : :
98 [ + - ]: 5670024 : set->cpl = *cpl;
99 [ + - + - ]: 5670024 : set->bserrno = 0;
100 [ + - + - ]: 5670024 : set->channel = channel;
101 [ + - + - ]: 5670024 : set->back_channel = back_channel;
102 : :
103 [ + - + - : 5670024 : set->cb_args.cb_fn = bs_sequence_completion;
+ - ]
104 [ + - + - : 5670024 : set->cb_args.cb_arg = set;
+ - ]
105 [ + - + - : 5670024 : set->cb_args.channel = channel->dev_channel;
+ - + - +
- ]
106 [ + - + - ]: 5670024 : set->ext_io_opts = NULL;
107 : :
108 : 5670024 : return (spdk_bs_sequence_t *)set;
109 : 18437 : }
110 : :
111 : : /* Use when performing IO directly on the blobstore (e.g. metadata - not a blob). */
112 : : spdk_bs_sequence_t *
113 : 123273 : bs_sequence_start_bs(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl)
114 : : {
115 : 123273 : return bs_sequence_start(_channel, cpl, _channel);
116 : : }
117 : :
118 : : /* Use when performing IO on a blob. */
119 : : spdk_bs_sequence_t *
120 : 5579994 : bs_sequence_start_blob(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl,
121 : : struct spdk_blob *blob)
122 : : {
123 : 5579994 : struct spdk_io_channel *esnap_ch = _channel;
124 : :
125 [ + + ]: 5579994 : if (spdk_blob_is_esnap_clone(blob)) {
126 : 19775 : esnap_ch = blob_esnap_get_io_channel(_channel, blob);
127 [ - + ]: 19775 : if (esnap_ch == NULL) {
128 : : /*
129 : : * The most likely reason we are here is because of some logic error
130 : : * elsewhere that caused channel allocations to fail. We could get here due
131 : : * to being out of memory as well. If we are out of memory, the process is
132 : : * this will be just one of many problems that this process will be having.
133 : : * Killing it off debug builds now due to logic errors is the right thing to
134 : : * do and killing it off due to ENOMEM is no big loss.
135 : : */
136 [ # # ]: 0 : assert(false);
137 : : return NULL;
138 : : }
139 : 1500 : }
140 : 5579994 : return bs_sequence_start(_channel, cpl, esnap_ch);
141 : : }
142 : :
143 : : void
144 : 1208 : bs_sequence_read_bs_dev(spdk_bs_sequence_t *seq, struct spdk_bs_dev *bs_dev,
145 : : void *payload, uint64_t lba, uint32_t lba_count,
146 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
147 : : {
148 : 1208 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
149 [ # # # # ]: 1208 : struct spdk_io_channel *back_channel = set->back_channel;
150 : :
151 [ + + - + : 1208 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
152 : : lba);
153 : :
154 [ # # # # : 1208 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
155 [ # # # # : 1208 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
156 : :
157 [ # # # # : 1208 : bs_dev->read(bs_dev, back_channel, payload, lba, lba_count, &set->cb_args);
# # # # #
# ]
158 : 1208 : }
159 : :
160 : : void
161 : 72352 : bs_sequence_read_dev(spdk_bs_sequence_t *seq, void *payload,
162 : : uint64_t lba, uint32_t lba_count,
163 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
164 : : {
165 : 72352 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
166 [ + - + - ]: 72352 : struct spdk_bs_channel *channel = set->channel;
167 : :
168 [ + + + + : 72352 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
+ - ]
169 : : lba);
170 : :
171 [ + - + - : 72352 : set->u.sequence.cb_fn = cb_fn;
+ - + - ]
172 [ + - + - : 72352 : set->u.sequence.cb_arg = cb_arg;
+ - + - ]
173 : :
174 [ + - + - : 72352 : channel->dev->read(channel->dev, channel->dev_channel, payload, lba, lba_count, &set->cb_args);
+ - + - -
+ + - + -
+ - + - +
- + - ]
175 : 72352 : }
176 : :
177 : : void
178 : 111011 : bs_sequence_write_dev(spdk_bs_sequence_t *seq, void *payload,
179 : : uint64_t lba, uint32_t lba_count,
180 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
181 : : {
182 : 111011 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
183 [ # # # # ]: 111011 : struct spdk_bs_channel *channel = set->channel;
184 : :
185 [ + + - + : 111011 : SPDK_DEBUGLOG(blob_rw, "Writing %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
186 : : lba);
187 : :
188 [ # # # # : 111011 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
189 [ # # # # : 111011 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
190 : :
191 [ # # # # : 119598 : channel->dev->write(channel->dev, channel->dev_channel, payload, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
192 [ # # ]: 8587 : &set->cb_args);
193 : 111011 : }
194 : :
195 : : void
196 : 755198 : bs_sequence_readv_bs_dev(spdk_bs_sequence_t *seq, struct spdk_bs_dev *bs_dev,
197 : : struct iovec *iov, int iovcnt, uint64_t lba, uint32_t lba_count,
198 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
199 : : {
200 : 755198 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
201 [ # # # # ]: 755198 : struct spdk_io_channel *back_channel = set->back_channel;
202 : :
203 [ + + - + : 755198 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
204 : : lba);
205 : :
206 [ # # # # : 755198 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
207 [ # # # # : 755198 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
208 : :
209 [ + + # # : 755198 : if (set->ext_io_opts) {
# # ]
210 [ - + # # : 749310 : assert(bs_dev->readv_ext);
# # # # ]
211 [ # # # # : 749397 : bs_dev->readv_ext(bs_dev, back_channel, iov, iovcnt, lba, lba_count,
# # # # ]
212 [ # # # # : 87 : &set->cb_args, set->ext_io_opts);
# # ]
213 : 87 : } else {
214 [ # # # # : 5888 : bs_dev->readv(bs_dev, back_channel, iov, iovcnt, lba, lba_count, &set->cb_args);
# # # # #
# ]
215 : : }
216 : 755198 : }
217 : :
218 : : void
219 : 1628487 : bs_sequence_readv_dev(spdk_bs_sequence_t *seq, struct iovec *iov, int iovcnt,
220 : : uint64_t lba, uint32_t lba_count, spdk_bs_sequence_cpl cb_fn, void *cb_arg)
221 : : {
222 : 1628487 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
223 [ # # # # ]: 1628487 : struct spdk_bs_channel *channel = set->channel;
224 : :
225 [ + + - + : 1628487 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
226 : : lba);
227 : :
228 [ # # # # : 1628487 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
229 [ # # # # : 1628487 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
230 [ + + # # : 1628487 : if (set->ext_io_opts) {
# # ]
231 [ - + # # : 1627191 : assert(channel->dev->readv_ext);
# # # # #
# # # ]
232 [ # # # # : 1627425 : channel->dev->readv_ext(channel->dev, channel->dev_channel, iov, iovcnt, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
233 [ # # # # : 234 : &set->cb_args, set->ext_io_opts);
# # ]
234 : 234 : } else {
235 [ # # # # : 1296 : channel->dev->readv(channel->dev, channel->dev_channel, iov, iovcnt, lba, lba_count, &set->cb_args);
# # # # #
# # # # #
# # # # #
# # # ]
236 : : }
237 : 1628487 : }
238 : :
239 : : void
240 : 3157233 : bs_sequence_writev_dev(spdk_bs_sequence_t *seq, struct iovec *iov, int iovcnt,
241 : : uint64_t lba, uint32_t lba_count,
242 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
243 : : {
244 : 3157233 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
245 [ # # # # ]: 3157233 : struct spdk_bs_channel *channel = set->channel;
246 : :
247 [ + + - + : 3157233 : SPDK_DEBUGLOG(blob_rw, "Writing %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
248 : : lba);
249 : :
250 [ # # # # : 3157233 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
251 [ # # # # : 3157233 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
252 : :
253 [ + + # # : 3157233 : if (set->ext_io_opts) {
# # ]
254 [ - + # # : 3153808 : assert(channel->dev->writev_ext);
# # # # #
# # # ]
255 [ # # # # : 3153880 : channel->dev->writev_ext(channel->dev, channel->dev_channel, iov, iovcnt, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
256 [ # # # # : 72 : &set->cb_args, set->ext_io_opts);
# # ]
257 : 72 : } else {
258 [ # # # # : 3629 : channel->dev->writev(channel->dev, channel->dev_channel, iov, iovcnt, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
259 [ # # ]: 204 : &set->cb_args);
260 : : }
261 : 3157233 : }
262 : :
263 : : void
264 : 129 : bs_sequence_write_zeroes_dev(spdk_bs_sequence_t *seq,
265 : : uint64_t lba, uint64_t lba_count,
266 : : spdk_bs_sequence_cpl cb_fn, void *cb_arg)
267 : : {
268 : 129 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
269 [ # # # # ]: 129 : struct spdk_bs_channel *channel = set->channel;
270 : :
271 [ + + - + : 129 : SPDK_DEBUGLOG(blob_rw, "writing zeroes to %" PRIu64 " blocks at LBA %" PRIu64 "\n",
# # ]
272 : : lba_count, lba);
273 : :
274 [ # # # # : 129 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
275 [ # # # # : 129 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
276 : :
277 [ # # # # : 134 : channel->dev->write_zeroes(channel->dev, channel->dev_channel, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
278 [ # # ]: 5 : &set->cb_args);
279 : 129 : }
280 : :
281 : : void
282 : 574 : bs_sequence_copy_dev(spdk_bs_sequence_t *seq, uint64_t dst_lba, uint64_t src_lba,
283 : : uint64_t lba_count, spdk_bs_sequence_cpl cb_fn, void *cb_arg)
284 : : {
285 : 574 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
286 [ # # # # ]: 574 : struct spdk_bs_channel *channel = set->channel;
287 : :
288 [ + + - + : 574 : SPDK_DEBUGLOG(blob_rw, "Copying %" PRIu64 " blocks from LBA %" PRIu64 " to LBA %" PRIu64 "\n",
# # ]
289 : : lba_count, src_lba, dst_lba);
290 : :
291 [ # # # # : 574 : set->u.sequence.cb_fn = cb_fn;
# # # # ]
292 [ # # # # : 574 : set->u.sequence.cb_arg = cb_arg;
# # # # ]
293 : :
294 [ # # # # : 574 : channel->dev->copy(channel->dev, channel->dev_channel, dst_lba, src_lba, lba_count, &set->cb_args);
# # # # #
# # # # #
# # # # #
# # # ]
295 : 574 : }
296 : :
297 : : void
298 : 5670024 : bs_sequence_finish(spdk_bs_sequence_t *seq, int bserrno)
299 : : {
300 [ + + ]: 5670024 : if (bserrno != 0) {
301 [ + - + - ]: 5804 : seq->bserrno = bserrno;
302 : 452 : }
303 : 5670024 : bs_request_set_complete((struct spdk_bs_request_set *)seq);
304 : 5670024 : }
305 : :
306 : : void
307 : 0 : bs_user_op_sequence_finish(void *cb_arg, int bserrno)
308 : : {
309 : 0 : spdk_bs_sequence_t *seq = cb_arg;
310 : :
311 : 0 : bs_sequence_finish(seq, bserrno);
312 : 0 : }
313 : :
314 : : static void
315 : 1795109 : bs_batch_completion(struct spdk_io_channel *_channel,
316 : : void *cb_arg, int bserrno)
317 : : {
318 : 1795109 : struct spdk_bs_request_set *set = cb_arg;
319 : :
320 [ # # # # : 1795109 : set->u.batch.outstanding_ops--;
# # ]
321 [ + + ]: 1795109 : if (bserrno != 0) {
322 [ # # # # ]: 16 : set->bserrno = bserrno;
323 : 4 : }
324 : :
325 [ + + + + : 1795109 : if (set->u.batch.outstanding_ops == 0 && set->u.batch.batch_closed) {
# # # # #
# # # # #
# # # # #
# ]
326 [ + + # # : 1641114 : if (set->u.batch.cb_fn) {
# # # # #
# ]
327 [ # # # # : 17654 : set->cb_args.cb_fn = bs_sequence_completion;
# # ]
328 [ # # # # : 17654 : set->u.batch.cb_fn((spdk_bs_sequence_t *)set, set->u.batch.cb_arg, bserrno);
# # # # #
# # # # #
# # # # #
# ]
329 : 3948 : } else {
330 : 1623460 : bs_request_set_complete(set);
331 : : }
332 : 40520 : }
333 : 1795109 : }
334 : :
335 : : spdk_bs_batch_t *
336 : 1773076 : bs_batch_open(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl, struct spdk_blob *blob)
337 : : {
338 : : struct spdk_bs_channel *channel;
339 : : struct spdk_bs_request_set *set;
340 : 1773076 : struct spdk_io_channel *back_channel = _channel;
341 : :
342 [ + + ]: 1773076 : if (spdk_blob_is_esnap_clone(blob)) {
343 : 29360 : back_channel = blob_esnap_get_io_channel(_channel, blob);
344 [ + + ]: 29360 : if (back_channel == NULL) {
345 : 0 : return NULL;
346 : : }
347 : 7340 : }
348 : :
349 : 1773076 : channel = spdk_io_channel_get_ctx(_channel);
350 [ + + # # ]: 1773076 : assert(channel != NULL);
351 [ # # # # : 1773076 : set = TAILQ_FIRST(&channel->reqs);
# # ]
352 [ + + ]: 1773076 : if (!set) {
353 : 0 : return NULL;
354 : : }
355 [ + - # # : 1773076 : TAILQ_REMOVE(&channel->reqs, set, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
356 : :
357 [ + + - + : 1773076 : spdk_trace_record(TRACE_BLOB_REQ_SET_START, 0, 0, (uintptr_t)&set->cb_args,
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
358 : : (uintptr_t)cpl->u.blob_basic.cb_arg);
359 : :
360 [ # # ]: 1773076 : set->cpl = *cpl;
361 [ # # # # ]: 1773076 : set->bserrno = 0;
362 [ # # # # ]: 1773076 : set->channel = channel;
363 [ # # # # ]: 1773076 : set->back_channel = back_channel;
364 : :
365 [ # # # # : 1773076 : set->u.batch.cb_fn = NULL;
# # # # ]
366 [ # # # # : 1773076 : set->u.batch.cb_arg = NULL;
# # # # ]
367 [ # # # # : 1773076 : set->u.batch.outstanding_ops = 0;
# # # # ]
368 [ # # # # : 1773076 : set->u.batch.batch_closed = 0;
# # # # ]
369 : :
370 [ # # # # : 1773076 : set->cb_args.cb_fn = bs_batch_completion;
# # ]
371 [ # # # # : 1773076 : set->cb_args.cb_arg = set;
# # ]
372 [ # # # # : 1773076 : set->cb_args.channel = channel->dev_channel;
# # # # #
# ]
373 : :
374 : 1773076 : return (spdk_bs_batch_t *)set;
375 : 37508 : }
376 : :
377 : : void
378 : 4352 : bs_batch_read_bs_dev(spdk_bs_batch_t *batch, struct spdk_bs_dev *bs_dev,
379 : : void *payload, uint64_t lba, uint32_t lba_count)
380 : : {
381 : 4352 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
382 [ # # # # ]: 4352 : struct spdk_io_channel *back_channel = set->back_channel;
383 : :
384 [ + + - + : 4352 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
385 : : lba);
386 : :
387 [ # # # # : 4352 : set->u.batch.outstanding_ops++;
# # ]
388 [ # # # # : 4352 : bs_dev->read(bs_dev, back_channel, payload, lba, lba_count, &set->cb_args);
# # # # #
# ]
389 : 4352 : }
390 : :
391 : : void
392 : 1314345 : bs_batch_read_dev(spdk_bs_batch_t *batch, void *payload,
393 : : uint64_t lba, uint32_t lba_count)
394 : : {
395 : 1314345 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
396 [ # # # # ]: 1314345 : struct spdk_bs_channel *channel = set->channel;
397 : :
398 [ + + - + : 1314345 : SPDK_DEBUGLOG(blob_rw, "Reading %" PRIu32 " blocks from LBA %" PRIu64 "\n", lba_count,
# # ]
399 : : lba);
400 : :
401 [ # # # # : 1314345 : set->u.batch.outstanding_ops++;
# # ]
402 [ # # # # : 1314345 : channel->dev->read(channel->dev, channel->dev_channel, payload, lba, lba_count, &set->cb_args);
# # # # #
# # # # #
# # # # #
# # # ]
403 : 1314345 : }
404 : :
405 : : void
406 : 301632 : bs_batch_write_dev(spdk_bs_batch_t *batch, void *payload,
407 : : uint64_t lba, uint32_t lba_count)
408 : : {
409 : 301632 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
410 [ # # # # ]: 301632 : struct spdk_bs_channel *channel = set->channel;
411 : :
412 [ + + - + : 301632 : SPDK_DEBUGLOG(blob_rw, "Writing %" PRIu32 " blocks to LBA %" PRIu64 "\n", lba_count, lba);
# # ]
413 : :
414 [ # # # # : 301632 : set->u.batch.outstanding_ops++;
# # ]
415 [ # # # # : 322203 : channel->dev->write(channel->dev, channel->dev_channel, payload, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
416 [ # # ]: 20571 : &set->cb_args);
417 : 301632 : }
418 : :
419 : : void
420 : 161709 : bs_batch_unmap_dev(spdk_bs_batch_t *batch,
421 : : uint64_t lba, uint64_t lba_count)
422 : : {
423 : 161709 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
424 [ # # # # ]: 161709 : struct spdk_bs_channel *channel = set->channel;
425 : :
426 [ + + - + : 161709 : SPDK_DEBUGLOG(blob_rw, "Unmapping %" PRIu64 " blocks at LBA %" PRIu64 "\n", lba_count,
# # ]
427 : : lba);
428 : :
429 [ # # # # : 161709 : set->u.batch.outstanding_ops++;
# # ]
430 [ # # # # : 163533 : channel->dev->unmap(channel->dev, channel->dev_channel, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
431 [ # # ]: 1824 : &set->cb_args);
432 : 161709 : }
433 : :
434 : : void
435 : 13071 : bs_batch_write_zeroes_dev(spdk_bs_batch_t *batch,
436 : : uint64_t lba, uint64_t lba_count)
437 : : {
438 : 13071 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
439 [ # # # # ]: 13071 : struct spdk_bs_channel *channel = set->channel;
440 : :
441 [ + + - + : 13071 : SPDK_DEBUGLOG(blob_rw, "Zeroing %" PRIu64 " blocks at LBA %" PRIu64 "\n", lba_count, lba);
# # ]
442 : :
443 [ # # # # : 13071 : set->u.batch.outstanding_ops++;
# # ]
444 [ # # # # : 15770 : channel->dev->write_zeroes(channel->dev, channel->dev_channel, lba, lba_count,
# # # # #
# # # # #
# # # # #
# ]
445 [ # # ]: 2699 : &set->cb_args);
446 : 13071 : }
447 : :
448 : : void
449 : 2032837 : bs_batch_close(spdk_bs_batch_t *batch)
450 : : {
451 : 2032837 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)batch;
452 : :
453 [ # # # # : 2032837 : set->u.batch.batch_closed = 1;
# # # # ]
454 : :
455 [ + + # # : 2032837 : if (set->u.batch.outstanding_ops == 0) {
# # # # #
# ]
456 [ + + # # : 391723 : if (set->u.batch.cb_fn) {
# # # # #
# ]
457 [ # # # # : 242107 : set->cb_args.cb_fn = bs_sequence_completion;
# # ]
458 [ # # # # : 242107 : set->u.batch.cb_fn((spdk_bs_sequence_t *)set, set->u.batch.cb_arg, set->bserrno);
# # # # #
# # # # #
# # # # #
# # # #
# ]
459 : 15464 : } else {
460 : 149616 : bs_request_set_complete(set);
461 : : }
462 : 16400 : }
463 : 2032837 : }
464 : :
465 : : spdk_bs_batch_t *
466 : 259761 : bs_sequence_to_batch(spdk_bs_sequence_t *seq, spdk_bs_sequence_cpl cb_fn, void *cb_arg)
467 : : {
468 : 259761 : struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)seq;
469 : :
470 [ # # # # : 259761 : set->u.batch.cb_fn = cb_fn;
# # # # ]
471 [ # # # # : 259761 : set->u.batch.cb_arg = cb_arg;
# # # # ]
472 [ # # # # : 259761 : set->u.batch.outstanding_ops = 0;
# # # # ]
473 [ # # # # : 259761 : set->u.batch.batch_closed = 0;
# # # # ]
474 : :
475 [ # # # # : 259761 : set->cb_args.cb_fn = bs_batch_completion;
# # ]
476 : :
477 : 259761 : return set;
478 : : }
479 : :
480 : : spdk_bs_user_op_t *
481 : 22488 : bs_user_op_alloc(struct spdk_io_channel *_channel, struct spdk_bs_cpl *cpl,
482 : : enum spdk_blob_op_type op_type, struct spdk_blob *blob,
483 : : void *payload, int iovcnt, uint64_t offset, uint64_t length)
484 : : {
485 : : struct spdk_bs_channel *channel;
486 : : struct spdk_bs_request_set *set;
487 : : struct spdk_bs_user_op_args *args;
488 : :
489 : 22488 : channel = spdk_io_channel_get_ctx(_channel);
490 [ + + # # ]: 22488 : assert(channel != NULL);
491 [ # # # # : 22488 : set = TAILQ_FIRST(&channel->reqs);
# # ]
492 [ + + ]: 22488 : if (!set) {
493 : 0 : return NULL;
494 : : }
495 [ + - # # : 22488 : TAILQ_REMOVE(&channel->reqs, set, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # ]
496 : :
497 [ + + - + : 22488 : spdk_trace_record(TRACE_BLOB_REQ_SET_START, 0, 0, (uintptr_t)&set->cb_args,
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
498 : : (uintptr_t)cpl->u.blob_basic.cb_arg);
499 : :
500 [ # # ]: 22488 : set->cpl = *cpl;
501 [ # # # # ]: 22488 : set->channel = channel;
502 [ # # # # ]: 22488 : set->back_channel = NULL;
503 [ # # # # ]: 22488 : set->ext_io_opts = NULL;
504 : :
505 [ # # # # ]: 22488 : args = &set->u.user_op;
506 : :
507 [ # # # # ]: 22488 : args->type = op_type;
508 [ # # # # ]: 22488 : args->iovcnt = iovcnt;
509 [ # # # # ]: 22488 : args->blob = blob;
510 [ # # # # ]: 22488 : args->offset = offset;
511 [ # # # # ]: 22488 : args->length = length;
512 [ # # # # ]: 22488 : args->payload = payload;
513 : :
514 : 22488 : return (spdk_bs_user_op_t *)set;
515 : 824 : }
516 : :
517 : : void
518 : 22487 : bs_user_op_execute(spdk_bs_user_op_t *op)
519 : : {
520 : : struct spdk_bs_request_set *set;
521 : : struct spdk_bs_user_op_args *args;
522 : : struct spdk_io_channel *ch;
523 : :
524 : 22487 : set = (struct spdk_bs_request_set *)op;
525 [ # # # # ]: 22487 : args = &set->u.user_op;
526 [ # # # # ]: 22487 : ch = spdk_io_channel_from_ctx(set->channel);
527 : :
528 [ + + - - : 22487 : switch (args->type) {
- + - # #
# # ]
529 : 1394 : case SPDK_BLOB_READ:
530 [ # # # # : 2298 : spdk_blob_io_read(args->blob, ch, args->payload, args->offset, args->length,
# # # # #
# # # # #
# # ]
531 [ # # # # : 452 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg);
# # # # #
# # # # #
# # # # #
# ]
532 : 1846 : break;
533 : 1187 : case SPDK_BLOB_WRITE:
534 [ # # # # : 1899 : spdk_blob_io_write(args->blob, ch, args->payload, args->offset, args->length,
# # # # #
# # # # #
# # ]
535 [ # # # # : 356 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg);
# # # # #
# # # # #
# # # # #
# ]
536 : 1543 : break;
537 : 0 : case SPDK_BLOB_UNMAP:
538 [ # # # # : 0 : spdk_blob_io_unmap(args->blob, ch, args->offset, args->length,
# # # # #
# # # ]
539 [ # # # # : 0 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg);
# # # # #
# # # # #
# # # # #
# ]
540 : 0 : break;
541 : 0 : case SPDK_BLOB_WRITE_ZEROES:
542 [ # # # # : 0 : spdk_blob_io_write_zeroes(args->blob, ch, args->offset, args->length,
# # # # #
# # # ]
543 [ # # # # : 0 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg);
# # # # #
# # # # #
# # # # #
# ]
544 : 0 : break;
545 : 0 : case SPDK_BLOB_READV:
546 [ # # # # : 0 : spdk_blob_io_readv_ext(args->blob, ch, args->payload, args->iovcnt,
# # # # #
# # # ]
547 [ # # # # : 0 : args->offset, args->length,
# # # # ]
548 [ # # # # : 0 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
549 [ # # # # ]: 0 : set->ext_io_opts);
550 : 0 : break;
551 : 19082 : case SPDK_BLOB_WRITEV:
552 [ # # # # : 19114 : spdk_blob_io_writev_ext(args->blob, ch, args->payload, args->iovcnt,
# # # # #
# # # ]
553 [ # # # # : 16 : args->offset, args->length,
# # # # ]
554 [ # # # # : 16 : set->cpl.u.blob_basic.cb_fn, set->cpl.u.blob_basic.cb_arg,
# # # # #
# # # # #
# # # # #
# ]
555 [ # # # # ]: 16 : set->ext_io_opts);
556 : 19098 : break;
557 : : }
558 [ # # # # : 22487 : TAILQ_INSERT_TAIL(&set->channel->reqs, set, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
559 : 22487 : }
560 : :
561 : : void
562 : 1 : bs_user_op_abort(spdk_bs_user_op_t *op, int bserrno)
563 : : {
564 : : struct spdk_bs_request_set *set;
565 : :
566 : 1 : set = (struct spdk_bs_request_set *)op;
567 : :
568 [ # # # # : 1 : set->cpl.u.blob_basic.cb_fn(set->cpl.u.blob_basic.cb_arg, bserrno);
# # # # #
# # # # #
# # # # #
# # # #
# ]
569 [ # # # # : 1 : TAILQ_INSERT_TAIL(&set->channel->reqs, set, link);
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # ]
570 : 1 : }
571 : :
572 : 2137 : SPDK_LOG_REGISTER_COMPONENT(blob_rw)
|