Branch data Line data Source code
1 : : /* SPDX-License-Identifier: BSD-3-Clause
2 : : * Copyright (C) 2016 Intel Corporation. All rights reserved.
3 : : * Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
4 : : * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : : */
6 : :
7 : : #ifndef __NVMF_INTERNAL_H__
8 : : #define __NVMF_INTERNAL_H__
9 : :
10 : : #include "spdk/stdinc.h"
11 : :
12 : : #include "spdk/keyring.h"
13 : : #include "spdk/likely.h"
14 : : #include "spdk/nvmf.h"
15 : : #include "spdk/nvmf_cmd.h"
16 : : #include "spdk/nvmf_transport.h"
17 : : #include "spdk/nvmf_spec.h"
18 : : #include "spdk/assert.h"
19 : : #include "spdk/bdev.h"
20 : : #include "spdk/queue.h"
21 : : #include "spdk/util.h"
22 : : #include "spdk/thread.h"
23 : : #include "spdk/tree.h"
24 : : #include "spdk/bit_array.h"
25 : :
26 : : /* The spec reserves cntlid values in the range FFF0h to FFFFh. */
27 : : #define NVMF_MIN_CNTLID 1
28 : : #define NVMF_MAX_CNTLID 0xFFEF
29 : :
30 : : enum spdk_nvmf_tgt_state {
31 : : NVMF_TGT_IDLE = 0,
32 : : NVMF_TGT_RUNNING,
33 : : NVMF_TGT_PAUSING,
34 : : NVMF_TGT_PAUSED,
35 : : NVMF_TGT_RESUMING,
36 : : };
37 : :
38 : : enum spdk_nvmf_subsystem_state {
39 : : SPDK_NVMF_SUBSYSTEM_INACTIVE = 0,
40 : : SPDK_NVMF_SUBSYSTEM_ACTIVATING,
41 : : SPDK_NVMF_SUBSYSTEM_ACTIVE,
42 : : SPDK_NVMF_SUBSYSTEM_PAUSING,
43 : : SPDK_NVMF_SUBSYSTEM_PAUSED,
44 : : SPDK_NVMF_SUBSYSTEM_RESUMING,
45 : : SPDK_NVMF_SUBSYSTEM_DEACTIVATING,
46 : : SPDK_NVMF_SUBSYSTEM_NUM_STATES,
47 : : };
48 : :
49 : : RB_HEAD(subsystem_tree, spdk_nvmf_subsystem);
50 : :
51 : : struct spdk_nvmf_tgt {
52 : : char name[NVMF_TGT_NAME_MAX_LENGTH];
53 : :
54 : : pthread_mutex_t mutex;
55 : :
56 : : uint64_t discovery_genctr;
57 : :
58 : : uint32_t max_subsystems;
59 : :
60 : : uint32_t discovery_filter;
61 : :
62 : : enum spdk_nvmf_tgt_state state;
63 : :
64 : : struct spdk_bit_array *subsystem_ids;
65 : :
66 : : struct subsystem_tree subsystems;
67 : :
68 : : TAILQ_HEAD(, spdk_nvmf_transport) transports;
69 : : TAILQ_HEAD(, spdk_nvmf_poll_group) poll_groups;
70 : : TAILQ_HEAD(, spdk_nvmf_referral) referrals;
71 : :
72 : : /* Used for round-robin assignment of connections to poll groups */
73 : : struct spdk_nvmf_poll_group *next_poll_group;
74 : :
75 : : spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn;
76 : : void *destroy_cb_arg;
77 : :
78 : : uint16_t crdt[3];
79 : : uint16_t num_poll_groups;
80 : :
81 : : /* Allowed DH-HMAC-CHAP digests/dhgroups */
82 : : uint32_t dhchap_digests;
83 : : uint32_t dhchap_dhgroups;
84 : :
85 : : TAILQ_ENTRY(spdk_nvmf_tgt) link;
86 : : };
87 : :
88 : : struct spdk_nvmf_host {
89 : : char nqn[SPDK_NVMF_NQN_MAX_LEN + 1];
90 : : struct spdk_key *dhchap_key;
91 : : struct spdk_key *dhchap_ctrlr_key;
92 : : TAILQ_ENTRY(spdk_nvmf_host) link;
93 : : };
94 : :
95 : : struct spdk_nvmf_subsystem_listener {
96 : : struct spdk_nvmf_subsystem *subsystem;
97 : : spdk_nvmf_tgt_subsystem_listen_done_fn cb_fn;
98 : : void *cb_arg;
99 : : struct spdk_nvme_transport_id *trid;
100 : : struct spdk_nvmf_transport *transport;
101 : : enum spdk_nvme_ana_state *ana_state;
102 : : uint64_t ana_state_change_count;
103 : : uint16_t id;
104 : : struct spdk_nvmf_listener_opts opts;
105 : : TAILQ_ENTRY(spdk_nvmf_subsystem_listener) link;
106 : : };
107 : :
108 : : struct spdk_nvmf_referral {
109 : : /* Discovery Log Page Entry for this referral */
110 : : struct spdk_nvmf_discovery_log_page_entry entry;
111 : : /* Make referral visible to controllers of these hosts */
112 : : TAILQ_HEAD(, spdk_nvmf_host) hosts;
113 : : /* This mutex is used to protect fields that aren't touched on the I/O path (e.g. it's
114 : : * needed for handling things like the CONNECT command) instead of requiring the subsystem
115 : : * to be paused. It makes it possible to modify those fields (e.g. add/remove hosts)
116 : : * without affecting outstanding I/O requests.
117 : : */
118 : : pthread_mutex_t mutex;
119 : : /* Transport ID */
120 : : struct spdk_nvme_transport_id trid;
121 : : struct spdk_nvmf_tgt *tgt;
122 : : /* Protected against concurrent access by ->mutex */
123 : : bool allow_any_host;
124 : : TAILQ_ENTRY(spdk_nvmf_referral) link;
125 : : };
126 : :
127 : : struct spdk_nvmf_subsystem_pg_ns_info {
128 : : struct spdk_io_channel *channel;
129 : : struct spdk_uuid uuid;
130 : : /* current reservation key, no reservation if the value is 0 */
131 : : uint64_t crkey;
132 : : /* reservation type */
133 : : enum spdk_nvme_reservation_type rtype;
134 : : /* Host ID which holds the reservation */
135 : : struct spdk_uuid holder_id;
136 : : /* Host ID for the registrants with the namespace */
137 : : struct spdk_uuid reg_hostid[SPDK_NVMF_MAX_NUM_REGISTRANTS];
138 : : uint64_t num_blocks;
139 : : uint32_t anagrpid;
140 : :
141 : : /* I/O outstanding to this namespace */
142 : : uint64_t io_outstanding;
143 : : enum spdk_nvmf_subsystem_state state;
144 : : };
145 : :
146 : : typedef void(*spdk_nvmf_poll_group_mod_done)(void *cb_arg, int status);
147 : :
148 : : struct spdk_nvmf_subsystem_poll_group {
149 : : /* Array of namespace information for each namespace indexed by nsid - 1 */
150 : : struct spdk_nvmf_subsystem_pg_ns_info *ns_info;
151 : : uint32_t num_ns;
152 : : enum spdk_nvmf_subsystem_state state;
153 : :
154 : : /* Number of ADMIN and FABRICS requests outstanding */
155 : : uint64_t mgmt_io_outstanding;
156 : : spdk_nvmf_poll_group_mod_done cb_fn;
157 : : void *cb_arg;
158 : :
159 : : TAILQ_HEAD(, spdk_nvmf_request) queued;
160 : : };
161 : :
162 : : struct spdk_nvmf_registrant {
163 : : TAILQ_ENTRY(spdk_nvmf_registrant) link;
164 : : struct spdk_uuid hostid;
165 : : /* Registration key */
166 : : uint64_t rkey;
167 : : };
168 : :
169 : : struct spdk_nvmf_ns {
170 : : uint32_t nsid;
171 : : uint32_t anagrpid;
172 : : struct spdk_nvmf_subsystem *subsystem;
173 : : struct spdk_bdev *bdev;
174 : : struct spdk_bdev_desc *desc;
175 : : struct spdk_nvmf_ns_opts opts;
176 : : /* reservation notification mask */
177 : : uint32_t mask;
178 : : /* generation code */
179 : : uint32_t gen;
180 : : /* registrants head */
181 : : TAILQ_HEAD(, spdk_nvmf_registrant) registrants;
182 : : /* current reservation key */
183 : : uint64_t crkey;
184 : : /* reservation type */
185 : : enum spdk_nvme_reservation_type rtype;
186 : : /* current reservation holder, only valid if reservation type can only have one holder */
187 : : struct spdk_nvmf_registrant *holder;
188 : : /* Persist Through Power Loss file which contains the persistent reservation */
189 : : char *ptpl_file;
190 : : /* Persist Through Power Loss feature is enabled */
191 : : bool ptpl_activated;
192 : : /* ZCOPY supported on bdev device */
193 : : bool zcopy;
194 : : /* Command Set Identifier */
195 : : enum spdk_nvme_csi csi;
196 : : /* Make namespace visible to controllers of these hosts */
197 : : TAILQ_HEAD(, spdk_nvmf_host) hosts;
198 : : /* Namespace is always visible to all controllers */
199 : : bool always_visible;
200 : : /* Namespace id of the underlying device, used for passthrough commands */
201 : : uint32_t passthru_nsid;
202 : : };
203 : :
204 : : /*
205 : : * NVMf reservation notification log page.
206 : : */
207 : : struct spdk_nvmf_reservation_log {
208 : : struct spdk_nvme_reservation_notification_log log;
209 : : TAILQ_ENTRY(spdk_nvmf_reservation_log) link;
210 : : struct spdk_nvmf_ctrlr *ctrlr;
211 : : };
212 : :
213 : : /*
214 : : * NVMf async event completion.
215 : : */
216 : : struct spdk_nvmf_async_event_completion {
217 : : union spdk_nvme_async_event_completion event;
218 : : STAILQ_ENTRY(spdk_nvmf_async_event_completion) link;
219 : : };
220 : :
221 : : /*
222 : : * This structure represents an NVMe-oF controller,
223 : : * which is like a "session" in networking terms.
224 : : */
225 : : struct spdk_nvmf_ctrlr {
226 : : uint16_t cntlid;
227 : : char hostnqn[SPDK_NVMF_NQN_MAX_LEN + 1];
228 : : struct spdk_nvmf_subsystem *subsys;
229 : : struct spdk_bit_array *visible_ns;
230 : :
231 : : struct spdk_nvmf_ctrlr_data cdata;
232 : :
233 : : struct spdk_nvmf_registers vcprop;
234 : :
235 : : struct spdk_nvmf_ctrlr_feat feat;
236 : :
237 : : struct spdk_nvmf_qpair *admin_qpair;
238 : : struct spdk_thread *thread;
239 : : struct spdk_bit_array *qpair_mask;
240 : :
241 : : const struct spdk_nvmf_subsystem_listener *listener;
242 : :
243 : : struct spdk_nvmf_request *aer_req[SPDK_NVMF_MAX_ASYNC_EVENTS];
244 : : STAILQ_HEAD(, spdk_nvmf_async_event_completion) async_events;
245 : : uint64_t notice_aen_mask;
246 : : uint8_t nr_aer_reqs;
247 : : struct spdk_uuid hostid;
248 : :
249 : : uint32_t association_timeout; /* in milliseconds */
250 : : uint16_t changed_ns_list_count;
251 : : struct spdk_nvme_ns_list changed_ns_list;
252 : : uint64_t log_page_count;
253 : : uint8_t num_avail_log_pages;
254 : : TAILQ_HEAD(log_page_head, spdk_nvmf_reservation_log) log_head;
255 : :
256 : : /* Time to trigger keep-alive--poller_time = now_tick + period */
257 : : uint64_t last_keep_alive_tick;
258 : : struct spdk_poller *keep_alive_poller;
259 : :
260 : : struct spdk_poller *association_timer;
261 : :
262 : : struct spdk_poller *cc_timer;
263 : : uint64_t cc_timeout_tsc;
264 : : struct spdk_poller *cc_timeout_timer;
265 : :
266 : : bool dif_insert_or_strip;
267 : : bool in_destruct;
268 : : bool disconnect_in_progress;
269 : : /* valid only when disconnect_in_progress is true */
270 : : bool disconnect_is_shn;
271 : : bool acre_enabled;
272 : : bool dynamic_ctrlr;
273 : : /* LBA Format Extension Enabled (LBAFEE) */
274 : : bool lbafee_enabled;
275 : :
276 : : TAILQ_ENTRY(spdk_nvmf_ctrlr) link;
277 : : };
278 : :
279 : : #define NVMF_MAX_LISTENERS_PER_SUBSYSTEM 16
280 : :
281 : : struct nvmf_subsystem_state_change_ctx {
282 : : struct spdk_nvmf_subsystem *subsystem;
283 : : uint16_t nsid;
284 : :
285 : : enum spdk_nvmf_subsystem_state original_state;
286 : : enum spdk_nvmf_subsystem_state requested_state;
287 : : int status;
288 : : struct spdk_thread *thread;
289 : :
290 : : spdk_nvmf_subsystem_state_change_done cb_fn;
291 : : void *cb_arg;
292 : : TAILQ_ENTRY(nvmf_subsystem_state_change_ctx) link;
293 : : };
294 : :
295 : : struct spdk_nvmf_subsystem {
296 : : struct spdk_thread *thread;
297 : :
298 : : uint32_t id;
299 : :
300 : : enum spdk_nvmf_subsystem_state state;
301 : : enum spdk_nvmf_subtype subtype;
302 : :
303 : : uint16_t next_cntlid;
304 : : struct {
305 : : uint8_t allow_any_listener : 1;
306 : : uint8_t ana_reporting : 1;
307 : : uint8_t reserved : 6;
308 : : } flags;
309 : :
310 : : /* Protected against concurrent access by ->mutex */
311 : : bool allow_any_host;
312 : :
313 : : bool destroying;
314 : : bool async_destroy;
315 : :
316 : : /* FDP related fields */
317 : : bool fdp_supported;
318 : :
319 : : /* Zoned storage related fields */
320 : : uint64_t max_zone_append_size_kib;
321 : :
322 : : struct spdk_nvmf_tgt *tgt;
323 : : RB_ENTRY(spdk_nvmf_subsystem) link;
324 : :
325 : : /* Array of pointers to namespaces of size max_nsid indexed by nsid - 1 */
326 : : struct spdk_nvmf_ns **ns;
327 : : uint32_t max_nsid;
328 : :
329 : : uint16_t min_cntlid;
330 : : uint16_t max_cntlid;
331 : :
332 : : uint64_t max_discard_size_kib;
333 : : uint64_t max_write_zeroes_size_kib;
334 : :
335 : : TAILQ_HEAD(, spdk_nvmf_ctrlr) ctrlrs;
336 : :
337 : : /* This mutex is used to protect fields that aren't touched on the I/O path (e.g. it's
338 : : * needed for handling things like the CONNECT command) instead of requiring the subsystem
339 : : * to be paused. It makes it possible to modify those fields (e.g. add/remove hosts)
340 : : * without affecting outstanding I/O requests.
341 : : */
342 : : pthread_mutex_t mutex;
343 : : /* Protected against concurrent access by ->mutex */
344 : : TAILQ_HEAD(, spdk_nvmf_host) hosts;
345 : : TAILQ_HEAD(, spdk_nvmf_subsystem_listener) listeners;
346 : : struct spdk_bit_array *used_listener_ids;
347 : :
348 : : TAILQ_ENTRY(spdk_nvmf_subsystem) entries;
349 : :
350 : : nvmf_subsystem_destroy_cb async_destroy_cb;
351 : : void *async_destroy_cb_arg;
352 : :
353 : : char sn[SPDK_NVME_CTRLR_SN_LEN + 1];
354 : : char mn[SPDK_NVME_CTRLR_MN_LEN + 1];
355 : : char subnqn[SPDK_NVMF_NQN_MAX_LEN + 1];
356 : :
357 : : /* Array of namespace count per ANA group of size max_nsid indexed anagrpid - 1
358 : : * It will be enough for ANA group to use the same size as namespaces.
359 : : */
360 : : uint32_t *ana_group;
361 : : /* Queue of a state change requests */
362 : : TAILQ_HEAD(, nvmf_subsystem_state_change_ctx) state_changes;
363 : : /* In-band authentication sequence number, protected by ->mutex */
364 : : uint32_t auth_seqnum;
365 : : bool passthrough;
366 : : };
367 : :
368 : : static int
369 : 101226 : subsystem_cmp(struct spdk_nvmf_subsystem *subsystem1, struct spdk_nvmf_subsystem *subsystem2)
370 : : {
371 [ + + + + : 101226 : return strncmp(subsystem1->subnqn, subsystem2->subnqn, sizeof(subsystem1->subnqn));
+ - + - ]
372 : : }
373 : :
374 [ + + + + : 130615 : RB_GENERATE_STATIC(subsystem_tree, spdk_nvmf_subsystem, link, subsystem_cmp);
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + -
+ + + - +
- + + + +
+ + + + -
- - - - -
+ - + - +
- + + + -
+ - + + -
+ + - + -
+ - - + #
# # # + -
+ + + - +
+ + + + +
+ - + + +
+ + - - +
# # # # #
# + - + -
+ - + + +
- + - + -
+ - + - +
- + + + -
+ - + - +
- + - + -
+ + + - +
- + - + -
+ - + - +
+ + - + -
+ - + - +
- + - + -
+ - + - +
+ + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
+ - # # #
# # # + -
+ - + - +
- + - + -
+ + + - +
- + - + -
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # +
- + - + -
+ - + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# # # # #
# # # # #
# # # + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - - + +
- + - # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - + -
+ - + - +
- + - - +
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# + - + -
+ - + + +
- + - + -
+ - + - +
- + - + -
- + + - +
- + - + -
+ - + - #
# # # # #
# # # # #
# + + + -
+ - + - +
- + - + -
+ + + + +
- + - + -
+ - + - +
- + - + -
# # # # #
# # # # #
# # # # +
- + - + -
- + + - +
- + - - +
+ - + - +
- # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # # # #
# # ]
375 : :
376 : : int nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group,
377 : : struct spdk_nvmf_subsystem *subsystem);
378 : : int nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group,
379 : : struct spdk_nvmf_subsystem *subsystem,
380 : : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg);
381 : : void nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group,
382 : : struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg);
383 : : void nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group,
384 : : struct spdk_nvmf_subsystem *subsystem,
385 : : uint32_t nsid,
386 : : spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg);
387 : : void nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group,
388 : : struct spdk_nvmf_subsystem *subsystem, spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg);
389 : :
390 : : void nvmf_get_discovery_log_page(struct spdk_nvmf_tgt *tgt, const char *hostnqn, struct iovec *iov,
391 : : uint32_t iovcnt, uint64_t offset, uint32_t length,
392 : : struct spdk_nvme_transport_id *cmd_source_trid);
393 : :
394 : : void nvmf_ctrlr_destruct(struct spdk_nvmf_ctrlr *ctrlr);
395 : : int nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req);
396 : : int nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req);
397 : : bool nvmf_ctrlr_dsm_supported(struct spdk_nvmf_ctrlr *ctrlr);
398 : : bool nvmf_ctrlr_write_zeroes_supported(struct spdk_nvmf_ctrlr *ctrlr);
399 : : bool nvmf_ctrlr_copy_supported(struct spdk_nvmf_ctrlr *ctrlr);
400 : : void nvmf_ctrlr_ns_changed(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid);
401 : : bool nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req);
402 : :
403 : : void nvmf_bdev_ctrlr_identify_ns(struct spdk_nvmf_ns *ns, struct spdk_nvme_ns_data *nsdata,
404 : : bool dif_insert_or_strip);
405 : : void nvmf_bdev_ctrlr_identify_iocs_nvm(struct spdk_nvmf_ns *ns,
406 : : struct spdk_nvme_nvm_ns_data *nsdata_nvm);
407 : : int nvmf_bdev_ctrlr_read_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
408 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
409 : : int nvmf_bdev_ctrlr_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
410 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
411 : : int nvmf_bdev_ctrlr_compare_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
412 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
413 : : int nvmf_bdev_ctrlr_compare_and_write_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
414 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *cmp_req, struct spdk_nvmf_request *write_req);
415 : : int nvmf_bdev_ctrlr_write_zeroes_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
416 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
417 : : int nvmf_bdev_ctrlr_flush_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
418 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
419 : : int nvmf_bdev_ctrlr_dsm_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
420 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
421 : : int nvmf_bdev_ctrlr_copy_cmd(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
422 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
423 : : int nvmf_bdev_ctrlr_nvme_passthru_io(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
424 : : struct spdk_io_channel *ch, struct spdk_nvmf_request *req);
425 : : bool nvmf_bdev_ctrlr_get_dif_ctx(struct spdk_bdev *bdev, struct spdk_nvme_cmd *cmd,
426 : : struct spdk_dif_ctx *dif_ctx);
427 : : bool nvmf_bdev_zcopy_enabled(struct spdk_bdev *bdev);
428 : :
429 : : int nvmf_subsystem_add_ctrlr(struct spdk_nvmf_subsystem *subsystem,
430 : : struct spdk_nvmf_ctrlr *ctrlr);
431 : : void nvmf_subsystem_remove_ctrlr(struct spdk_nvmf_subsystem *subsystem,
432 : : struct spdk_nvmf_ctrlr *ctrlr);
433 : : void nvmf_subsystem_remove_all_listeners(struct spdk_nvmf_subsystem *subsystem,
434 : : bool stop);
435 : : struct spdk_nvmf_ctrlr *nvmf_subsystem_get_ctrlr(struct spdk_nvmf_subsystem *subsystem,
436 : : uint16_t cntlid);
437 : : bool nvmf_subsystem_host_auth_required(struct spdk_nvmf_subsystem *subsystem, const char *hostnqn);
438 : : enum nvmf_auth_key_type {
439 : : NVMF_AUTH_KEY_HOST,
440 : : NVMF_AUTH_KEY_CTRLR,
441 : : };
442 : : struct spdk_key *nvmf_subsystem_get_dhchap_key(struct spdk_nvmf_subsystem *subsys, const char *nqn,
443 : : enum nvmf_auth_key_type type);
444 : : struct spdk_nvmf_subsystem_listener *nvmf_subsystem_find_listener(
445 : : struct spdk_nvmf_subsystem *subsystem,
446 : : const struct spdk_nvme_transport_id *trid);
447 : : bool nvmf_subsystem_zone_append_supported(struct spdk_nvmf_subsystem *subsystem);
448 : : struct spdk_nvmf_listener *nvmf_transport_find_listener(
449 : : struct spdk_nvmf_transport *transport,
450 : : const struct spdk_nvme_transport_id *trid);
451 : : void nvmf_transport_dump_opts(struct spdk_nvmf_transport *transport, struct spdk_json_write_ctx *w,
452 : : bool named);
453 : : void nvmf_transport_listen_dump_trid(const struct spdk_nvme_transport_id *trid,
454 : : struct spdk_json_write_ctx *w);
455 : :
456 : : int nvmf_ctrlr_async_event_ns_notice(struct spdk_nvmf_ctrlr *ctrlr);
457 : : int nvmf_ctrlr_async_event_ana_change_notice(struct spdk_nvmf_ctrlr *ctrlr);
458 : : void nvmf_ctrlr_async_event_discovery_log_change_notice(void *ctx);
459 : : void nvmf_ctrlr_async_event_reservation_notification(struct spdk_nvmf_ctrlr *ctrlr);
460 : :
461 : : void nvmf_ns_reservation_request(void *ctx);
462 : : void nvmf_ctrlr_reservation_notice_log(struct spdk_nvmf_ctrlr *ctrlr,
463 : : struct spdk_nvmf_ns *ns,
464 : : enum spdk_nvme_reservation_notification_log_page_type type);
465 : :
466 : : bool nvmf_ns_is_ptpl_capable(const struct spdk_nvmf_ns *ns);
467 : :
468 : : static inline struct spdk_nvmf_host *
469 : 208 : nvmf_ns_find_host(struct spdk_nvmf_ns *ns, const char *hostnqn)
470 : : {
471 : 208 : struct spdk_nvmf_host *host = NULL;
472 : :
473 [ + + # # : 226 : TAILQ_FOREACH(host, &ns->hosts, link) {
# # # # #
# # # #
# ]
474 [ + + - + : 65 : if (strcmp(hostnqn, host->nqn) == 0) {
+ + # # ]
475 : 47 : return host;
476 : : }
477 : 3 : }
478 : :
479 : 161 : return NULL;
480 : 43 : }
481 : :
482 : : /*
483 : : * Abort zero-copy requests that already got the buffer (received zcopy_start cb), but haven't
484 : : * started zcopy_end. These requests are kept on the outstanding queue, but are not waiting for a
485 : : * completion from the bdev layer, so, when a qpair is being disconnected, we need to kick them to
486 : : * force their completion.
487 : : */
488 : : void nvmf_qpair_abort_pending_zcopy_reqs(struct spdk_nvmf_qpair *qpair);
489 : :
490 : : /*
491 : : * Free aer simply frees the rdma resources for the aer without informing the host.
492 : : * This function should be called when deleting a qpair when one wants to make sure
493 : : * the qpair is completely empty before freeing the request. The reason we free the
494 : : * AER without sending a completion is to prevent the host from sending another AER.
495 : : */
496 : : void nvmf_qpair_free_aer(struct spdk_nvmf_qpair *qpair);
497 : :
498 : : int nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req);
499 : :
500 : : void nvmf_ctrlr_set_fatal_status(struct spdk_nvmf_ctrlr *ctrlr);
501 : :
502 : : static inline bool
503 : 26040484 : nvmf_ctrlr_ns_is_visible(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid)
504 : : {
505 [ + - + - : 26040484 : assert(nsid > 0 && nsid <= ctrlr->subsys->max_nsid);
+ - + - +
- # # ]
506 [ + - + - ]: 26040484 : return spdk_bit_array_get(ctrlr->visible_ns, nsid - 1);
507 : : }
508 : :
509 : : static inline void
510 : 5583 : nvmf_ctrlr_ns_set_visible(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid, bool visible)
511 : : {
512 [ + - + - : 5583 : assert(nsid > 0 && nsid <= ctrlr->subsys->max_nsid);
+ - + - +
- # # ]
513 [ + + + - ]: 5583 : if (visible) {
514 [ + - + - ]: 3680 : spdk_bit_array_set(ctrlr->visible_ns, nsid - 1);
515 : 834 : } else {
516 [ # # # # ]: 1903 : spdk_bit_array_clear(ctrlr->visible_ns, nsid - 1);
517 : : }
518 : 5583 : }
519 : :
520 : : static inline struct spdk_nvmf_ns *
521 : 28632518 : _nvmf_subsystem_get_ns(struct spdk_nvmf_subsystem *subsystem, uint32_t nsid)
522 : : {
523 : : /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */
524 [ + + + - : 28632518 : if (spdk_unlikely(nsid - 1 >= subsystem->max_nsid)) {
+ + ]
525 : 45182 : return NULL;
526 : : }
527 : :
528 [ + - + - : 28587336 : return subsystem->ns[nsid - 1];
+ - + - ]
529 : 301603 : }
530 : :
531 : : static inline struct spdk_nvmf_ns *
532 : 26064393 : nvmf_ctrlr_get_ns(struct spdk_nvmf_ctrlr *ctrlr, uint32_t nsid)
533 : : {
534 [ + - + - ]: 26064393 : struct spdk_nvmf_subsystem *subsystem = ctrlr->subsys;
535 : 26064393 : struct spdk_nvmf_ns *ns = _nvmf_subsystem_get_ns(subsystem, nsid);
536 : :
537 [ + + + + ]: 26064393 : return ns && nvmf_ctrlr_ns_is_visible(ctrlr, nsid) ? ns : NULL;
538 : : }
539 : :
540 : : static inline bool
541 : 83613322 : nvmf_qpair_is_admin_queue(struct spdk_nvmf_qpair *qpair)
542 : : {
543 [ + - + - ]: 83613322 : return qpair->qid == 0;
544 : : }
545 : :
546 : : void nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, enum spdk_nvmf_qpair_state state);
547 : :
548 : : int nvmf_qpair_auth_init(struct spdk_nvmf_qpair *qpair);
549 : : void nvmf_qpair_auth_destroy(struct spdk_nvmf_qpair *qpair);
550 : : void nvmf_qpair_auth_dump(struct spdk_nvmf_qpair *qpair, struct spdk_json_write_ctx *w);
551 : :
552 : : int nvmf_auth_request_exec(struct spdk_nvmf_request *req);
553 : : bool nvmf_auth_is_supported(void);
554 : :
555 : : static inline bool
556 : 22129 : nvmf_request_is_fabric_connect(struct spdk_nvmf_request *req)
557 : : {
558 [ + + + - : 44222 : return req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC &&
+ - + - +
- - + ]
559 [ + - + - : 22093 : req->cmd->nvmf_cmd.fctype == SPDK_NVMF_FABRIC_COMMAND_CONNECT;
+ - + - +
- ]
560 : : }
561 : :
562 : : /*
563 : : * Tests whether a given string represents a valid NQN.
564 : : */
565 : : bool nvmf_nqn_is_valid(const char *nqn);
566 : :
567 : : /*
568 : : * Tests whether a given NQN describes a discovery subsystem.
569 : : */
570 : : bool nvmf_nqn_is_discovery(const char *nqn);
571 : :
572 : : /**
573 : : * Initiates a zcopy start operation
574 : : *
575 : : * \param bdev The \ref spdk_bdev
576 : : * \param desc The \ref spdk_bdev_desc
577 : : * \param ch The \ref spdk_io_channel
578 : : * \param req The \ref spdk_nvmf_request passed to the bdev for processing
579 : : *
580 : : * \return SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE if the command was completed immediately or
581 : : * SPDK_NVMF_REQUEST_EXEC_STATUS_ASYNCHRONOUS if the command was submitted and will be
582 : : * completed asynchronously. Asynchronous completions are notified through
583 : : * spdk_nvmf_request_complete().
584 : : */
585 : : int nvmf_bdev_ctrlr_zcopy_start(struct spdk_bdev *bdev,
586 : : struct spdk_bdev_desc *desc,
587 : : struct spdk_io_channel *ch,
588 : : struct spdk_nvmf_request *req);
589 : :
590 : : /**
591 : : * Ends a zcopy operation
592 : : *
593 : : * \param req The NVMe-oF request
594 : : * \param commit Flag indicating whether the buffers should be committed
595 : : */
596 : : void nvmf_bdev_ctrlr_zcopy_end(struct spdk_nvmf_request *req, bool commit);
597 : :
598 : : /**
599 : : * Publishes the mDNS PRR (Pull Registration Request) for the NVMe-oF target.
600 : : *
601 : : * \param tgt The NVMe-oF target
602 : : *
603 : : * \return 0 on success, negative errno on failure
604 : : */
605 : : int nvmf_publish_mdns_prr(struct spdk_nvmf_tgt *tgt);
606 : :
607 : : /**
608 : : * Stops the mDNS PRR (Pull Registration Request) for the NVMe-oF target.
609 : : *
610 : : * \param tgt The NVMe-oF target
611 : : */
612 : : void nvmf_tgt_stop_mdns_prr(struct spdk_nvmf_tgt *tgt);
613 : :
614 : : /**
615 : : * Updates the listener list in the mDNS PRR (Pull Registration Request) for the NVMe-oF target.
616 : : *
617 : : * \param tgt The NVMe-oF target
618 : : *
619 : : * \return 0 on success, negative errno on failure
620 : : */
621 : : int nvmf_tgt_update_mdns_prr(struct spdk_nvmf_tgt *tgt);
622 : :
623 : : static inline struct spdk_nvmf_transport_poll_group *
624 : 30755 : nvmf_get_transport_poll_group(struct spdk_nvmf_poll_group *group,
625 : : struct spdk_nvmf_transport *transport)
626 : : {
627 : : struct spdk_nvmf_transport_poll_group *tgroup;
628 : :
629 [ + + + - : 30755 : TAILQ_FOREACH(tgroup, &group->tgroups, link) {
+ - + + #
# # # #
# ]
630 [ + - + - : 30044 : if (tgroup->transport == transport) {
- + ]
631 : 30044 : return tgroup;
632 : : }
633 : 0 : }
634 : :
635 : 711 : return NULL;
636 : 3211 : }
637 : :
638 : : /**
639 : : * Generates a new NVMF controller id
640 : : *
641 : : * \param subsystem The subsystem
642 : : *
643 : : * \return unique controller id or 0xFFFF when all controller ids are in use
644 : : */
645 : : uint16_t nvmf_subsystem_gen_cntlid(struct spdk_nvmf_subsystem *subsystem);
646 : :
647 : : #endif /* __NVMF_INTERNAL_H__ */
|