Line data Source code
1 : /* SPDX-License-Identifier: BSD-3-Clause
2 : * Copyright (C) 2016 Intel Corporation.
3 : * All rights reserved.
4 : * Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5 : */
6 :
7 : /** \file
8 : * Thread
9 : */
10 :
11 : #ifndef SPDK_THREAD_H_
12 : #define SPDK_THREAD_H_
13 :
14 : #ifdef __linux__
15 : #include <sys/epoll.h>
16 : #endif
17 :
18 : #include "spdk/config.h"
19 : #include "spdk/fd_group.h"
20 : #include "spdk/stdinc.h"
21 : #include "spdk/assert.h"
22 : #include "spdk/cpuset.h"
23 : #include "spdk/env.h"
24 : #include "spdk/util.h"
25 : #include "spdk/likely.h"
26 :
27 : #ifdef __cplusplus
28 : extern "C" {
29 : #endif
30 :
31 : /**
32 : * Pollers should always return a value of this type
33 : * indicating whether they did real work or not.
34 : */
35 : enum spdk_thread_poller_rc {
36 : SPDK_POLLER_IDLE,
37 : SPDK_POLLER_BUSY,
38 : };
39 :
40 : /**
41 : * A stackless, lightweight thread.
42 : */
43 : struct spdk_thread;
44 :
45 : /**
46 : * A function repeatedly called on the same spdk_thread.
47 : */
48 : struct spdk_poller;
49 :
50 : struct spdk_io_channel_iter;
51 :
52 : /**
53 : * A function that is called each time a new thread is created.
54 : * The implementer of this function should frequently call
55 : * spdk_thread_poll() on the thread provided.
56 : *
57 : * \param thread The new spdk_thread.
58 : */
59 : typedef int (*spdk_new_thread_fn)(struct spdk_thread *thread);
60 :
61 : /**
62 : * SPDK thread operation type.
63 : */
64 : enum spdk_thread_op {
65 : /* Called each time a new thread is created. The implementer of this operation
66 : * should frequently call spdk_thread_poll() on the thread provided.
67 : */
68 : SPDK_THREAD_OP_NEW,
69 :
70 : /* Called when SPDK thread needs to be rescheduled. (e.g., when cpumask of the
71 : * SPDK thread is updated.
72 : */
73 : SPDK_THREAD_OP_RESCHED,
74 : };
75 :
76 : /**
77 : * Function to be called for SPDK thread operation.
78 : */
79 : typedef int (*spdk_thread_op_fn)(struct spdk_thread *thread, enum spdk_thread_op op);
80 :
81 : /**
82 : * Function to check whether the SPDK thread operation is supported.
83 : */
84 : typedef bool (*spdk_thread_op_supported_fn)(enum spdk_thread_op op);
85 :
86 : /**
87 : * A function that will be called on the target thread.
88 : *
89 : * \param ctx Context passed as arg to spdk_thread_pass_msg().
90 : */
91 : typedef void (*spdk_msg_fn)(void *ctx);
92 :
93 : /**
94 : * Function to be called to pass a message to a thread.
95 : *
96 : * \param fn Callback function for a thread.
97 : * \param ctx Context passed to fn.
98 : * \param thread_ctx Context for the thread.
99 : */
100 : typedef void (*spdk_thread_pass_msg)(spdk_msg_fn fn, void *ctx,
101 : void *thread_ctx);
102 :
103 : /**
104 : * Callback function for a poller.
105 : *
106 : * \param ctx Context passed as arg to spdk_poller_register().
107 : * \return value of type `enum spdk_thread_poller_rc` (ex: SPDK_POLLER_IDLE
108 : * if no work was done or SPDK_POLLER_BUSY if work was done.)
109 : */
110 : typedef int (*spdk_poller_fn)(void *ctx);
111 :
112 : /**
113 : * Callback function to set poller into interrupt mode or back to poll mode.
114 : *
115 : * \param poller Poller to set interrupt or poll mode.
116 : * \param cb_arg Argument passed to the callback function.
117 : * \param interrupt_mode Set interrupt mode for true, or poll mode for false
118 : */
119 : typedef void (*spdk_poller_set_interrupt_mode_cb)(struct spdk_poller *poller, void *cb_arg,
120 : bool interrupt_mode);
121 :
122 : /**
123 : * Mark that the poller is capable of entering interrupt mode.
124 : *
125 : * When registering the poller set interrupt callback, the callback will get
126 : * executed immediately if its spdk_thread is in the interrupt mode.
127 : *
128 : * Callers may pass NULL for the cb_fn, signifying that no callback is
129 : * necessary when the interrupt mode changes.
130 : *
131 : * \param poller The poller to register callback function.
132 : * \param cb_fn Callback function called when the poller must transition into or out of interrupt mode
133 : * \param cb_arg Argument passed to the callback function.
134 : */
135 : void spdk_poller_register_interrupt(struct spdk_poller *poller,
136 : spdk_poller_set_interrupt_mode_cb cb_fn,
137 : void *cb_arg);
138 :
139 : /**
140 : * I/O channel creation callback.
141 : *
142 : * \param io_device I/O device associated with this channel.
143 : * \param ctx_buf Context for the I/O device.
144 : */
145 : typedef int (*spdk_io_channel_create_cb)(void *io_device, void *ctx_buf);
146 :
147 : /**
148 : * I/O channel destruction callback.
149 : *
150 : * \param io_device I/O device associated with this channel.
151 : * \param ctx_buf Context for the I/O device.
152 : */
153 : typedef void (*spdk_io_channel_destroy_cb)(void *io_device, void *ctx_buf);
154 :
155 : /**
156 : * I/O device unregister callback.
157 : *
158 : * \param io_device Unregistered I/O device.
159 : */
160 : typedef void (*spdk_io_device_unregister_cb)(void *io_device);
161 :
162 : /**
163 : * Called on the appropriate thread for each channel associated with io_device.
164 : *
165 : * \param i I/O channel iterator.
166 : */
167 : typedef void (*spdk_channel_msg)(struct spdk_io_channel_iter *i);
168 :
169 : /**
170 : * spdk_for_each_channel() callback.
171 : *
172 : * \param i I/O channel iterator.
173 : * \param status 0 if it completed successfully, or negative errno if it failed.
174 : */
175 : typedef void (*spdk_channel_for_each_cpl)(struct spdk_io_channel_iter *i, int status);
176 :
177 : #define SPDK_IO_CHANNEL_STRUCT_SIZE 96
178 :
179 : /**
180 : * Message memory pool size definitions
181 : */
182 : #define SPDK_MSG_MEMPOOL_CACHE_SIZE 1024
183 : /* Power of 2 minus 1 is optimal for memory consumption */
184 : #define SPDK_DEFAULT_MSG_MEMPOOL_SIZE (262144 - 1)
185 :
186 : /**
187 : * Initialize the threading library. Must be called once prior to allocating any threads.
188 : *
189 : * \param new_thread_fn Called each time a new SPDK thread is created. The implementer
190 : * is expected to frequently call spdk_thread_poll() on the provided thread.
191 : * \param ctx_sz For each thread allocated, an additional region of memory of
192 : * size ctx_size will also be allocated, for use by the thread scheduler. A pointer
193 : * to this region may be obtained by calling spdk_thread_get_ctx().
194 : *
195 : * \return 0 on success. Negated errno on failure.
196 : */
197 : int spdk_thread_lib_init(spdk_new_thread_fn new_thread_fn, size_t ctx_sz);
198 :
199 : /**
200 : * Initialize the threading library. Must be called once prior to allocating any threads
201 : *
202 : * Both thread_op_fn and thread_op_type_supported_fn have to be specified or not
203 : * specified together.
204 : *
205 : * \param thread_op_fn Called for SPDK thread operation.
206 : * \param thread_op_supported_fn Called to check whether the SPDK thread operation is supported.
207 : * \param ctx_sz For each thread allocated, for use by the thread scheduler. A pointer
208 : * to this region may be obtained by calling spdk_thread_get_ctx().
209 : * \param msg_mempool_size Size of the allocated spdk_msg_mempool.
210 : *
211 : * \return 0 on success. Negated errno on failure.
212 : */
213 : int spdk_thread_lib_init_ext(spdk_thread_op_fn thread_op_fn,
214 : spdk_thread_op_supported_fn thread_op_supported_fn,
215 : size_t ctx_sz, size_t msg_mempool_size);
216 :
217 : /**
218 : * Release all resources associated with this library.
219 : */
220 : void spdk_thread_lib_fini(void);
221 :
222 : /**
223 : * Creates a new SPDK thread object.
224 : *
225 : * Note that the first thread created via spdk_thread_create() will be designated as
226 : * the app thread. Other SPDK libraries may place restrictions on certain APIs to
227 : * only be called in the context of this app thread.
228 : *
229 : * \param name Human-readable name for the thread; can be retrieved with spdk_thread_get_name().
230 : * The string is copied, so the pointed-to data only needs to be valid during the
231 : * spdk_thread_create() call. May be NULL to specify no name.
232 : * \param cpumask Optional mask of CPU cores on which to schedule this thread. This is only
233 : * a suggestion to the scheduler. The value is copied, so cpumask may be released when
234 : * this function returns. May be NULL if no mask is required.
235 : *
236 : * \return a pointer to the allocated thread on success or NULL on failure..
237 : */
238 : struct spdk_thread *spdk_thread_create(const char *name, const struct spdk_cpuset *cpumask);
239 :
240 : /**
241 : * Return the app thread.
242 : *
243 : * The app thread is the first thread created using spdk_thread_create().
244 : *
245 : * \return a pointer to the app thread, or NULL if no thread has been created yet.
246 : */
247 : struct spdk_thread *spdk_thread_get_app_thread(void);
248 :
249 : /**
250 : * Check if the specified spdk_thread is the app thread.
251 : *
252 : * \param thread The thread to check. If NULL, check the current spdk_thread.
253 : * \return true if the specified spdk_thread is the app thread, false otherwise.
254 : */
255 : bool spdk_thread_is_app_thread(struct spdk_thread *thread);
256 :
257 : /**
258 : * Force the current system thread to act as if executing the given SPDK thread.
259 : *
260 : * \param thread The thread to set.
261 : */
262 : void spdk_set_thread(struct spdk_thread *thread);
263 :
264 : /**
265 : * Bind or unbind spdk_thread to its current CPU core.
266 : *
267 : * If spdk_thread is bound, it couldn't be rescheduled to other CPU cores until it is unbound.
268 : *
269 : * \param thread The thread to bind or not.
270 : * \param bind true for bind, false for unbind.
271 : */
272 : void spdk_thread_bind(struct spdk_thread *thread, bool bind);
273 :
274 : /**
275 : * Returns whether the thread is bound to its current CPU core.
276 : *
277 : * \param thread The thread to query.
278 : *
279 : * \return true if bound, false otherwise
280 : */
281 : bool spdk_thread_is_bound(struct spdk_thread *thread);
282 :
283 : /**
284 : * Mark the thread as exited, failing all future spdk_thread_send_msg(),
285 : * spdk_poller_register(), and spdk_get_io_channel() calls. May only be called
286 : * within an spdk poller or message.
287 : *
288 : * All I/O channel references associated with the thread must be released
289 : * using spdk_put_io_channel(), and all active pollers associated with the thread
290 : * should be unregistered using spdk_poller_unregister(), prior to calling
291 : * this function. This function will complete these processing. The completion can
292 : * be queried by spdk_thread_is_exited().
293 : *
294 : * Note that this function must not be called on the app thread until after it
295 : * has been called for all other threads.
296 : *
297 : * \param thread The thread to exit.
298 : *
299 : * \return always 0. (return value was deprecated but keep it for ABI compatibility.)
300 : */
301 : int spdk_thread_exit(struct spdk_thread *thread);
302 :
303 : /**
304 : * Returns whether the thread is marked as exited.
305 : *
306 : * A thread is exited only after it has spdk_thread_exit() called on it, and
307 : * it has been polled until any outstanding operations targeting this
308 : * thread have completed. This may include poller unregistrations, io channel
309 : * unregistrations, or outstanding spdk_thread_send_msg calls.
310 : *
311 : * \param thread The thread to query.
312 : *
313 : * \return true if marked as exited, false otherwise.
314 : */
315 : bool spdk_thread_is_exited(struct spdk_thread *thread);
316 :
317 : /**
318 : * Returns whether the thread is still running.
319 : *
320 : * A thread is considered running until it has * spdk_thread_exit() called on it.
321 : *
322 : * \param thread The thread to query.
323 : *
324 : * \return true if still running, false otherwise.
325 : */
326 : bool spdk_thread_is_running(struct spdk_thread *thread);
327 :
328 : /**
329 : * Destroy a thread, releasing all of its resources. May only be called
330 : * on a thread previously marked as exited.
331 : *
332 : * \param thread The thread to destroy.
333 : *
334 : */
335 : void spdk_thread_destroy(struct spdk_thread *thread);
336 :
337 : /**
338 : * Return a pointer to this thread's context.
339 : *
340 : * \param thread The thread on which to get the context.
341 : *
342 : * \return a pointer to the per-thread context, or NULL if there is
343 : * no per-thread context.
344 : */
345 : void *spdk_thread_get_ctx(struct spdk_thread *thread);
346 :
347 : /**
348 : * Get the thread's cpumask.
349 : *
350 : * \param thread The thread to get the cpumask for.
351 : *
352 : * \return cpuset pointer
353 : */
354 : struct spdk_cpuset *spdk_thread_get_cpumask(struct spdk_thread *thread);
355 :
356 : /**
357 : * Set the current thread's cpumask to the specified value. The thread may be
358 : * rescheduled to one of the CPUs specified in the cpumask.
359 : *
360 : * This API requires SPDK thread operation supports SPDK_THREAD_OP_RESCHED.
361 : *
362 : * \param cpumask The new cpumask for the thread.
363 : *
364 : * \return 0 on success, negated errno otherwise.
365 : */
366 : int spdk_thread_set_cpumask(struct spdk_cpuset *cpumask);
367 :
368 : /**
369 : * Return the thread object associated with the context handle previously
370 : * obtained by calling spdk_thread_get_ctx().
371 : *
372 : * \param ctx A context previously obtained by calling spdk_thread_get_ctx()
373 : *
374 : * \return The associated thread.
375 : */
376 : struct spdk_thread *spdk_thread_get_from_ctx(void *ctx);
377 :
378 : /**
379 : * Perform one iteration worth of processing on the thread. This includes
380 : * both expired and continuous pollers as well as messages. If the thread
381 : * has exited, return immediately.
382 : *
383 : * \param thread The thread to process
384 : * \param max_msgs The maximum number of messages that will be processed.
385 : * Use 0 to process the default number of messages (8).
386 : * \param now The current time, in ticks. Optional. If 0 is passed, this
387 : * function will call spdk_get_ticks() to get the current time.
388 : * The current time is used as start time and this function
389 : * will call spdk_get_ticks() at its end to know end time to
390 : * measure run time of this function.
391 : *
392 : * \return 1 if work was done. 0 if no work was done.
393 : */
394 : int spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs, uint64_t now);
395 :
396 : /**
397 : * Return the number of ticks until the next timed poller
398 : * would expire. Timed pollers are pollers for which
399 : * period_microseconds is greater than 0.
400 : *
401 : * \param thread The thread to check poller expiration times on
402 : *
403 : * \return Number of ticks. If no timed pollers, return 0.
404 : */
405 : uint64_t spdk_thread_next_poller_expiration(struct spdk_thread *thread);
406 :
407 : /**
408 : * Returns whether there are any active pollers (pollers for which
409 : * period_microseconds equals 0) registered to be run on the thread.
410 : *
411 : * \param thread The thread to check.
412 : *
413 : * \return 1 if there is at least one active poller, 0 otherwise.
414 : */
415 : int spdk_thread_has_active_pollers(struct spdk_thread *thread);
416 :
417 : /**
418 : * Returns whether there are any pollers registered to be run
419 : * on the thread.
420 : *
421 : * \param thread The thread to check.
422 : *
423 : * \return true if there is any active poller, false otherwise.
424 : */
425 : bool spdk_thread_has_pollers(struct spdk_thread *thread);
426 :
427 : /**
428 : * Returns whether there are scheduled operations to be run on the thread.
429 : *
430 : * \param thread The thread to check.
431 : *
432 : * \return true if there are no scheduled operations, false otherwise.
433 : */
434 : bool spdk_thread_is_idle(struct spdk_thread *thread);
435 :
436 : /**
437 : * Get count of allocated threads.
438 : */
439 : uint32_t spdk_thread_get_count(void);
440 :
441 : /**
442 : * Get a handle to the current thread.
443 : *
444 : * This handle may be passed to other threads and used as the target of
445 : * spdk_thread_send_msg().
446 : *
447 : * \sa spdk_io_channel_get_thread()
448 : *
449 : * \return a pointer to the current thread on success or NULL on failure.
450 : */
451 : struct spdk_thread *spdk_get_thread(void);
452 :
453 : /**
454 : * Get a thread's name.
455 : *
456 : * \param thread Thread to query.
457 : *
458 : * \return the name of the thread.
459 : */
460 : const char *spdk_thread_get_name(const struct spdk_thread *thread);
461 :
462 : /**
463 : * Get a thread's ID.
464 : *
465 : * \param thread Thread to query.
466 : *
467 : * \return the ID of the thread..
468 : */
469 : uint64_t spdk_thread_get_id(const struct spdk_thread *thread);
470 :
471 : /**
472 : * Get the thread by the ID.
473 : *
474 : * \param id ID of the thread.
475 : * \return Thread whose ID matches or NULL otherwise.
476 : */
477 : struct spdk_thread *spdk_thread_get_by_id(uint64_t id);
478 :
479 : struct spdk_thread_stats {
480 : uint64_t busy_tsc;
481 : uint64_t idle_tsc;
482 : };
483 :
484 : /**
485 : * Get statistics about the current thread.
486 : *
487 : * Copy cumulative thread stats values to the provided thread stats structure.
488 : *
489 : * \param stats User's thread_stats structure.
490 : */
491 : int spdk_thread_get_stats(struct spdk_thread_stats *stats);
492 :
493 : /**
494 : * Return the TSC value from the end of the last time this thread was polled.
495 : *
496 : * \param thread Thread to query. If NULL, use current thread.
497 : *
498 : * \return TSC value from the end of the last time this thread was polled.
499 : */
500 : uint64_t spdk_thread_get_last_tsc(struct spdk_thread *thread);
501 :
502 : /**
503 : * Send a message to the given thread.
504 : *
505 : * The message will be sent asynchronously - i.e. spdk_thread_send_msg will always return
506 : * prior to `fn` being called.
507 : *
508 : * \param thread The target thread.
509 : * \param fn This function will be called on the given thread.
510 : * \param ctx This context will be passed to fn when called.
511 : *
512 : * \return 0 on success
513 : * \return -ENOMEM if the message could not be allocated
514 : * \return -EIO if the message could not be sent to the destination thread
515 : */
516 : int spdk_thread_send_msg(const struct spdk_thread *thread, spdk_msg_fn fn, void *ctx);
517 :
518 : /**
519 : * Send a message to the given thread. Only one critical message can be outstanding at the same
520 : * time. It's intended to use this function in any cases that might interrupt the execution of the
521 : * application, such as signal handlers.
522 : *
523 : * The message will be sent asynchronously - i.e. spdk_thread_send_critical_msg will always return
524 : * prior to `fn` being called.
525 : *
526 : * \param thread The target thread.
527 : * \param fn This function will be called on the given thread.
528 : *
529 : * \return 0 on success
530 : * \return -EIO if the message could not be sent to the destination thread, due to an already
531 : * outstanding critical message
532 : */
533 : int spdk_thread_send_critical_msg(struct spdk_thread *thread, spdk_msg_fn fn);
534 :
535 : /**
536 : * Run the msg callback on the given thread. If this happens to be the current
537 : * thread, the callback is executed immediately; otherwise a message is sent to
538 : * the thread, and it's run asynchronously.
539 : *
540 : * \param thread The target thread.
541 : * \param fn This function will be called on the given thread.
542 : * \param ctx This context will be passed to fn when called.
543 : *
544 : * \return 0 on success
545 : * \return -ENOMEM if the message could not be allocated
546 : * \return -EIO if the message could not be sent to the destination thread
547 : */
548 : static inline int
549 116 : spdk_thread_exec_msg(const struct spdk_thread *thread, spdk_msg_fn fn, void *ctx)
550 : {
551 116 : assert(thread != NULL);
552 :
553 116 : if (spdk_get_thread() == thread) {
554 114 : fn(ctx);
555 114 : return 0;
556 : }
557 :
558 2 : return spdk_thread_send_msg(thread, fn, ctx);
559 : }
560 :
561 : /**
562 : * Send a message to each thread, serially.
563 : *
564 : * The message is sent asynchronously - i.e. spdk_for_each_thread will return
565 : * prior to `fn` being called on each thread.
566 : *
567 : * \param fn This is the function that will be called on each thread.
568 : * \param ctx This context will be passed to fn when called.
569 : * \param cpl This will be called on the originating thread after `fn` has been
570 : * called on each thread.
571 : */
572 : void spdk_for_each_thread(spdk_msg_fn fn, void *ctx, spdk_msg_fn cpl);
573 :
574 : /**
575 : * Set current spdk_thread into interrupt mode or back to poll mode.
576 : *
577 : * Only valid when thread interrupt facility is enabled by
578 : * spdk_interrupt_mode_enable().
579 : *
580 : * \param enable_interrupt Set interrupt mode for true, or poll mode for false
581 : */
582 : void spdk_thread_set_interrupt_mode(bool enable_interrupt);
583 :
584 : /**
585 : * Get trace id.
586 : *
587 : * \param thread Thread to get trace_id from.
588 : *
589 : * \return Trace id of the specified thread.
590 : */
591 : uint16_t spdk_thread_get_trace_id(struct spdk_thread *thread);
592 :
593 : /**
594 : * Register a poller on the current thread.
595 : *
596 : * The poller can be unregistered by calling spdk_poller_unregister().
597 : *
598 : * \param fn This function will be called every `period_microseconds`.
599 : * \param arg Argument passed to fn.
600 : * \param period_microseconds How often to call `fn`. If 0, call `fn` as often
601 : * as possible.
602 : *
603 : * \return a pointer to the poller registered on the current thread on success
604 : * or NULL on failure.
605 : */
606 : struct spdk_poller *spdk_poller_register(spdk_poller_fn fn,
607 : void *arg,
608 : uint64_t period_microseconds);
609 :
610 : /**
611 : * Register a poller on the current thread with arbitrary name.
612 : *
613 : * The poller can be unregistered by calling spdk_poller_unregister().
614 : *
615 : * \param fn This function will be called every `period_microseconds`.
616 : * \param arg Argument passed to fn.
617 : * \param period_microseconds How often to call `fn`. If 0, call `fn` as often
618 : * as possible.
619 : * \param name Human readable name for the poller. Pointer of the poller function
620 : * name is set if NULL.
621 : *
622 : * \return a pointer to the poller registered on the current thread on success
623 : * or NULL on failure.
624 : */
625 : struct spdk_poller *spdk_poller_register_named(spdk_poller_fn fn,
626 : void *arg,
627 : uint64_t period_microseconds,
628 : const char *name);
629 :
630 : /*
631 : * \brief Register a poller on the current thread with setting its name
632 : * to the string of the poller function name. The poller being registered
633 : * should return a value of type `enum spdk_thread_poller_rc`. See
634 : * \ref spdk_poller_fn for more information.
635 : */
636 : #define SPDK_POLLER_REGISTER(fn, arg, period_microseconds) \
637 : spdk_poller_register_named(fn, arg, period_microseconds, #fn)
638 :
639 : /**
640 : * Unregister a poller on the current thread.
641 : *
642 : * This function will also write NULL to the spdk_poller pointer pointed
643 : * to by ppoller, to help encourage a poller pointer not getting reused
644 : * after it has been unregistered.
645 : *
646 : * It is OK to pass a ppoller parameter that points to NULL, in this case
647 : * the function is a nop.
648 : *
649 : * \param ppoller The poller to unregister.
650 : */
651 : void spdk_poller_unregister(struct spdk_poller **ppoller);
652 :
653 : /**
654 : * Pause a poller on the current thread.
655 : *
656 : * The poller is not run until it is resumed with spdk_poller_resume(). It is
657 : * perfectly fine to pause an already paused poller.
658 : *
659 : * \param poller The poller to pause.
660 : */
661 : void spdk_poller_pause(struct spdk_poller *poller);
662 :
663 : /**
664 : * Resume a poller on the current thread.
665 : *
666 : * Resumes a poller paused with spdk_poller_pause(). It is perfectly fine to
667 : * resume an unpaused poller.
668 : *
669 : * \param poller The poller to resume.
670 : */
671 : void spdk_poller_resume(struct spdk_poller *poller);
672 :
673 : /**
674 : * Register the opaque io_device context as an I/O device.
675 : *
676 : * After an I/O device is registered, it can return I/O channels using the
677 : * spdk_get_io_channel() function.
678 : *
679 : * \param io_device The pointer to io_device context.
680 : * \param create_cb Callback function invoked to allocate any resources required
681 : * for a new I/O channel.
682 : * \param destroy_cb Callback function invoked to release the resources for an
683 : * I/O channel.
684 : * \param ctx_size The size of the context buffer allocated to store references
685 : * to allocated I/O channel resources.
686 : * \param name A string name for the device used only for debugging. Optional -
687 : * may be NULL.
688 : */
689 : void spdk_io_device_register(void *io_device, spdk_io_channel_create_cb create_cb,
690 : spdk_io_channel_destroy_cb destroy_cb, uint32_t ctx_size,
691 : const char *name);
692 :
693 : /**
694 : * Unregister the opaque io_device context as an I/O device.
695 : *
696 : * The actual unregistration might be deferred until all active I/O channels are
697 : * destroyed.
698 : *
699 : * \param io_device The pointer to io_device context.
700 : * \param unregister_cb An optional callback function invoked to release any
701 : * references to this I/O device.
702 : */
703 : void spdk_io_device_unregister(void *io_device, spdk_io_device_unregister_cb unregister_cb);
704 :
705 : /**
706 : * Get an I/O channel for the specified io_device to be used by the calling thread.
707 : *
708 : * The io_device context pointer specified must have previously been registered
709 : * using spdk_io_device_register(). If an existing I/O channel does not exist
710 : * yet for the given io_device on the calling thread, it will allocate an I/O
711 : * channel and invoke the create_cb function pointer specified in spdk_io_device_register().
712 : * If an I/O channel already exists for the given io_device on the calling thread,
713 : * its reference is returned rather than creating a new I/O channel.
714 : *
715 : * \param io_device The pointer to io_device context.
716 : *
717 : * \return a pointer to the I/O channel for this device on success or NULL on failure.
718 : */
719 : struct spdk_io_channel *spdk_get_io_channel(void *io_device);
720 :
721 : /**
722 : * Release a reference to an I/O channel. This happens asynchronously.
723 : *
724 : * This must be called on the same thread that called spdk_get_io_channel()
725 : * for the specified I/O channel. If this releases the last reference to the
726 : * I/O channel, The destroy_cb function specified in spdk_io_device_register()
727 : * will be invoked to release any associated resources.
728 : *
729 : * \param ch I/O channel to release a reference.
730 : */
731 : void spdk_put_io_channel(struct spdk_io_channel *ch);
732 :
733 : /**
734 : * Get the context buffer associated with an I/O channel.
735 : *
736 : * \param ch I/O channel.
737 : *
738 : * \return a pointer to the context buffer.
739 : */
740 : static inline void *
741 78378 : spdk_io_channel_get_ctx(struct spdk_io_channel *ch)
742 : {
743 78378 : if (spdk_unlikely(!ch)) {
744 0 : assert(false);
745 : return NULL;
746 : }
747 :
748 78378 : return (uint8_t *)ch + SPDK_IO_CHANNEL_STRUCT_SIZE;
749 : }
750 :
751 : /**
752 : * Get I/O channel from the context buffer. This is the inverse of
753 : * spdk_io_channel_get_ctx().
754 : *
755 : * \param ctx The pointer to the context buffer.
756 : *
757 : * \return a pointer to the I/O channel associated with the context buffer.
758 : */
759 : struct spdk_io_channel *spdk_io_channel_from_ctx(void *ctx);
760 :
761 : /**
762 : * Get the thread associated with an I/O channel.
763 : *
764 : * \param ch I/O channel.
765 : *
766 : * \return a pointer to the thread associated with the I/O channel
767 : */
768 : struct spdk_thread *spdk_io_channel_get_thread(struct spdk_io_channel *ch);
769 :
770 : /**
771 : * Call 'fn' on each channel associated with io_device.
772 : *
773 : * This happens asynchronously, so fn may be called after spdk_for_each_channel
774 : * returns. 'fn' will be called for each channel serially, such that two calls
775 : * to 'fn' will not overlap in time. After 'fn' has been called, call
776 : * spdk_for_each_channel_continue() to continue iterating.
777 : *
778 : * \param io_device 'fn' will be called on each channel associated with this io_device.
779 : * \param fn Called on the appropriate thread for each channel associated with io_device.
780 : * \param ctx Context buffer registered to spdk_io_channel_iter that can be obtained
781 : * form the function spdk_io_channel_iter_get_ctx().
782 : * \param cpl Called on the thread that spdk_for_each_channel was initially called
783 : * from when 'fn' has been called on each channel.
784 : */
785 : void spdk_for_each_channel(void *io_device, spdk_channel_msg fn, void *ctx,
786 : spdk_channel_for_each_cpl cpl);
787 :
788 : /**
789 : * Get io_device from the I/O channel iterator.
790 : *
791 : * \param i I/O channel iterator.
792 : *
793 : * \return a pointer to the io_device.
794 : */
795 : void *spdk_io_channel_iter_get_io_device(struct spdk_io_channel_iter *i);
796 :
797 : /**
798 : * Get I/O channel from the I/O channel iterator.
799 : *
800 : * \param i I/O channel iterator.
801 : *
802 : * \return a pointer to the I/O channel.
803 : */
804 : struct spdk_io_channel *spdk_io_channel_iter_get_channel(struct spdk_io_channel_iter *i);
805 :
806 : /**
807 : * Get context buffer from the I/O channel iterator.
808 : *
809 : * \param i I/O channel iterator.
810 : *
811 : * \return a pointer to the context buffer.
812 : */
813 : void *spdk_io_channel_iter_get_ctx(struct spdk_io_channel_iter *i);
814 :
815 : /**
816 : * Get the io_device for the specified I/O channel.
817 : *
818 : * \param ch I/O channel.
819 : *
820 : * \return a pointer to the io_device for the I/O channel
821 : */
822 : void *spdk_io_channel_get_io_device(struct spdk_io_channel *ch);
823 :
824 : /**
825 : * Helper function to iterate all channels for spdk_for_each_channel().
826 : *
827 : * \param i I/O channel iterator.
828 : * \param status Status for the I/O channel iterator;
829 : * for non 0 status remaining iterations are terminated.
830 : */
831 : void spdk_for_each_channel_continue(struct spdk_io_channel_iter *i, int status);
832 :
833 : /**
834 : * A representative for registered interrupt file descriptor.
835 : */
836 : struct spdk_interrupt;
837 :
838 : /**
839 : * Callback function registered for interrupt file descriptor.
840 : *
841 : * \param ctx Context passed as arg to spdk_interrupt_register().
842 : *
843 : * \return 0 to indicate that interrupt took place but no events were found;
844 : * positive to indicate that interrupt took place and some events were processed;
845 : * negative if no event information is provided.
846 : */
847 : typedef int (*spdk_interrupt_fn)(void *ctx);
848 :
849 : /**
850 : * Register an spdk_interrupt on the current thread.
851 : *
852 : * The provided function will be called any time a SPDK_INTERRUPT_EVENT_IN event
853 : * triggers on the associated file descriptor.
854 : *
855 : * \param efd File descriptor of the spdk_interrupt.
856 : * \param fn Called each time there are events in spdk_interrupt.
857 : * \param arg Function argument for fn.
858 : * \param name Human readable name for the spdk_interrupt. Pointer of the spdk_interrupt
859 : * name is set if NULL.
860 : *
861 : * \return a pointer to the spdk_interrupt registered on the current thread on success
862 : * or NULL on failure.
863 : */
864 : struct spdk_interrupt *spdk_interrupt_register(int efd, spdk_interrupt_fn fn,
865 : void *arg, const char *name);
866 :
867 : /**
868 : * Register an spdk_interrupt with specific event types on the current thread.
869 : *
870 : * The provided function will be called any time one of specified event types triggers on
871 : * the associated file descriptor.
872 : * Event types argument is a bit mask composed by ORing together
873 : * enum spdk_interrupt_event_types values.
874 : *
875 : * \param efd File descriptor of the spdk_interrupt.
876 : * \param events Event notification types.
877 : * \param fn Called each time there are events in spdk_interrupt.
878 : * \param arg Function argument for fn.
879 : * \param name Human readable name for the spdk_interrupt. Pointer of the spdk_interrupt
880 : * name is set if NULL.
881 : *
882 : * \return a pointer to the spdk_interrupt registered on the current thread on success
883 : * or NULL on failure.
884 : */
885 : struct spdk_interrupt *spdk_interrupt_register_for_events(int efd, uint32_t events,
886 : spdk_interrupt_fn fn, void *arg, const char *name);
887 :
888 : /*
889 : * \brief Register an spdk_interrupt on the current thread with setting its name
890 : * to the string of the spdk_interrupt function name.
891 : */
892 : #define SPDK_INTERRUPT_REGISTER(efd, fn, arg) \
893 : spdk_interrupt_register(efd, fn, arg, #fn)
894 :
895 : /*
896 : * \brief Register an spdk_interrupt on the current thread with specific event types
897 : * and with setting its name to the string of the spdk_interrupt function name.
898 : */
899 : #define SPDK_INTERRUPT_REGISTER_FOR_EVENTS(efd, events, fn, arg) \
900 : spdk_interrupt_register_for_events(efd, events, fn, arg, #fn)
901 :
902 : /**
903 : * Unregister an spdk_interrupt on the current thread.
904 : *
905 : * \param pintr The spdk_interrupt to unregister.
906 : */
907 : void spdk_interrupt_unregister(struct spdk_interrupt **pintr);
908 :
909 : enum spdk_interrupt_event_types {
910 : #ifdef __linux__
911 : SPDK_INTERRUPT_EVENT_IN = EPOLLIN,
912 : SPDK_INTERRUPT_EVENT_OUT = EPOLLOUT,
913 : SPDK_INTERRUPT_EVENT_ET = EPOLLET
914 : #else
915 : SPDK_INTERRUPT_EVENT_IN = 0x001,
916 : SPDK_INTERRUPT_EVENT_OUT = 0x004,
917 : SPDK_INTERRUPT_EVENT_ET = 1u << 31
918 : #endif
919 : };
920 :
921 : /**
922 : * Change the event_types associated with the spdk_interrupt on the current thread.
923 : *
924 : * \param intr The pointer to the spdk_interrupt registered on the current thread.
925 : * \param event_types New event_types for the spdk_interrupt.
926 : *
927 : * \return 0 if success or -errno if failed.
928 : */
929 : int spdk_interrupt_set_event_types(struct spdk_interrupt *intr,
930 : enum spdk_interrupt_event_types event_types);
931 :
932 : /**
933 : * Return a file descriptor that becomes ready whenever any of the registered
934 : * interrupt file descriptors are ready
935 : *
936 : * \param thread The thread to get.
937 : *
938 : * \return The spdk_interrupt fd of thread itself.
939 : */
940 : int spdk_thread_get_interrupt_fd(struct spdk_thread *thread);
941 :
942 : /**
943 : * Return an fd_group that becomes ready whenever any of the registered
944 : * interrupt file descriptors are ready
945 : *
946 : *
947 : * \param thread The thread to get.
948 : *
949 : * \return The spdk_fd_group of the thread itself.
950 : */
951 : struct spdk_fd_group *spdk_thread_get_interrupt_fd_group(struct spdk_thread *thread);
952 :
953 : /**
954 : * Set SPDK run as event driven mode
955 : *
956 : * \return 0 on success or -errno on failure
957 : */
958 : int spdk_interrupt_mode_enable(void);
959 :
960 : /**
961 : * Reports whether interrupt mode is set.
962 : *
963 : * \return True if interrupt mode is set, false otherwise.
964 : */
965 : bool spdk_interrupt_mode_is_enabled(void);
966 :
967 : /**
968 : * A spinlock augmented with safety checks for use with SPDK.
969 : *
970 : * SPDK code that uses spdk_spinlock runs from an SPDK thread, which itself is associated with a
971 : * pthread. There are typically many SPDK threads associated with each pthread. The SPDK application
972 : * may migrate SPDK threads between pthreads from time to time to balance the load on those threads.
973 : * Migration of SPDK threads only happens when the thread is off CPU, and as such it is only safe to
974 : * hold a lock so long as an SPDK thread stays on CPU.
975 : *
976 : * It is not safe to lock a spinlock, return from the event or poller, then unlock it at some later
977 : * time because:
978 : *
979 : * - Even though the SPDK thread may be the same, the SPDK thread may be running on different
980 : * pthreads during lock and unlock. A pthread spinlock may consider this to be an unlock by a
981 : * non-owner, which results in undefined behavior.
982 : * - A lock that is acquired by a poller or event may be needed by another poller or event that
983 : * runs on the same pthread. This can lead to deadlock or detection of deadlock.
984 : * - A lock that is acquired by a poller or event that is needed by another poller or event that
985 : * runs on a second pthread will block the second pthread from doing any useful work until the
986 : * lock is released. Because the lock holder and the lock acquirer are on the same pthread, this
987 : * would lead to deadlock.
988 : *
989 : * If an SPDK spinlock is used erroneously, the program will abort.
990 : */
991 : struct spdk_spinlock {
992 : pthread_spinlock_t spinlock;
993 : struct spdk_thread *thread;
994 : struct spdk_spinlock_internal *internal;
995 : bool initialized;
996 : bool destroyed;
997 : };
998 :
999 : /**
1000 : * Initialize an spdk_spinlock.
1001 : *
1002 : * \param sspin The SPDK spinlock to initialize.
1003 : */
1004 : void spdk_spin_init(struct spdk_spinlock *sspin);
1005 :
1006 : /**
1007 : * Destroy an spdk_spinlock.
1008 : *
1009 : * \param sspin The SPDK spinlock to initialize.
1010 : */
1011 : void spdk_spin_destroy(struct spdk_spinlock *sspin);
1012 :
1013 : /**
1014 : * Lock an SPDK spin lock.
1015 : *
1016 : * \param sspin An SPDK spinlock.
1017 : */
1018 : void spdk_spin_lock(struct spdk_spinlock *sspin);
1019 :
1020 : /**
1021 : * Unlock an SPDK spinlock.
1022 : *
1023 : * \param sspin An SPDK spinlock.
1024 : */
1025 : void spdk_spin_unlock(struct spdk_spinlock *sspin);
1026 :
1027 : /**
1028 : * Determine if the caller holds this SPDK spinlock.
1029 : *
1030 : * \param sspin An SPDK spinlock.
1031 : * \return true if spinlock is held by this thread, else false
1032 : */
1033 : bool spdk_spin_held(struct spdk_spinlock *sspin);
1034 :
1035 : struct spdk_iobuf_opts {
1036 : /** Maximum number of small buffers */
1037 : uint64_t small_pool_count;
1038 : /** Maximum number of large buffers */
1039 : uint64_t large_pool_count;
1040 : /** Size of a single small buffer */
1041 : uint32_t small_bufsize;
1042 : /** Size of a single large buffer */
1043 : uint32_t large_bufsize;
1044 :
1045 : /**
1046 : * The size of spdk_iobuf_opts according to the caller of this library is used for ABI
1047 : * compatibility. The library uses this field to know how many fields in this
1048 : * structure are valid. And the library will populate any remaining fields with default values.
1049 : * New added fields should be put at the end of the struct.
1050 : */
1051 : size_t opts_size;
1052 :
1053 : /** Enable per-NUMA node buffer pools */
1054 : uint8_t enable_numa;
1055 : };
1056 :
1057 : struct spdk_iobuf_pool_stats {
1058 : /** Buffer got from local per-thread cache */
1059 : uint64_t cache;
1060 : /** Buffer got from the main shared pool */
1061 : uint64_t main;
1062 : /** Buffer missed and request to get buffer was queued */
1063 : uint64_t retry;
1064 : };
1065 :
1066 : struct spdk_iobuf_module_stats {
1067 : struct spdk_iobuf_pool_stats small_pool;
1068 : struct spdk_iobuf_pool_stats large_pool;
1069 : const char *module;
1070 : };
1071 :
1072 : struct spdk_iobuf_entry;
1073 :
1074 : typedef void (*spdk_iobuf_get_cb)(struct spdk_iobuf_entry *entry, void *buf);
1075 :
1076 : /** iobuf queue entry */
1077 : struct spdk_iobuf_entry {
1078 : spdk_iobuf_get_cb cb_fn;
1079 : const void *module;
1080 : STAILQ_ENTRY(spdk_iobuf_entry) stailq;
1081 : };
1082 :
1083 : struct spdk_iobuf_buffer {
1084 : STAILQ_ENTRY(spdk_iobuf_buffer) stailq;
1085 : };
1086 :
1087 : typedef STAILQ_HEAD(, spdk_iobuf_entry) spdk_iobuf_entry_stailq_t;
1088 : typedef STAILQ_HEAD(, spdk_iobuf_buffer) spdk_iobuf_buffer_stailq_t;
1089 :
1090 : struct spdk_iobuf_pool_cache {
1091 : /** Buffer pool */
1092 : struct spdk_ring *pool;
1093 : /** Buffer cache */
1094 : spdk_iobuf_buffer_stailq_t cache;
1095 : /** Number of elements in the cache */
1096 : uint32_t cache_count;
1097 : /** Size of the cache */
1098 : uint32_t cache_size;
1099 : /** Buffer wait queue */
1100 : spdk_iobuf_entry_stailq_t *queue;
1101 : /** Buffer size */
1102 : uint32_t bufsize;
1103 : /** Pool usage statistics */
1104 : struct spdk_iobuf_pool_stats stats;
1105 : };
1106 :
1107 : struct spdk_iobuf_node_cache {
1108 : /** Small buffer memory pool cache */
1109 : struct spdk_iobuf_pool_cache small;
1110 : /** Large buffer memory pool cache */
1111 : struct spdk_iobuf_pool_cache large;
1112 : };
1113 :
1114 : #ifndef SPDK_CONFIG_MAX_NUMA_NODES
1115 : /* Set this default temporarily, for users that may pull latest code without
1116 : * re-running configure.
1117 : */
1118 : #define SPDK_CONFIG_MAX_NUMA_NODES 1
1119 : #endif
1120 :
1121 : /** iobuf channel */
1122 : struct spdk_iobuf_channel {
1123 : /** Module pointer */
1124 : const void *module;
1125 : /** Parent IO channel */
1126 : struct spdk_io_channel *parent;
1127 : /* Buffer cache */
1128 : struct spdk_iobuf_node_cache cache[SPDK_CONFIG_MAX_NUMA_NODES];
1129 : };
1130 :
1131 : /**
1132 : * Initialize and allocate iobuf pools.
1133 : *
1134 : * \return 0 on success, negative errno otherwise.
1135 : */
1136 : int spdk_iobuf_initialize(void);
1137 :
1138 : typedef void (*spdk_iobuf_finish_cb)(void *cb_arg);
1139 :
1140 : /**
1141 : * Clean up and free iobuf pools.
1142 : *
1143 : * \param cb_fn Callback to be executed once the clean up is completed.
1144 : * \param cb_arg Callback argument.
1145 : */
1146 : void spdk_iobuf_finish(spdk_iobuf_finish_cb cb_fn, void *cb_arg);
1147 :
1148 : /**
1149 : * Set iobuf options. These options will be used during `spdk_iobuf_initialize()`.
1150 : *
1151 : * \param opts Options describing the size of the pools to reserve.
1152 : *
1153 : * \return 0 on success, negative errno otherwise.
1154 : */
1155 : int spdk_iobuf_set_opts(const struct spdk_iobuf_opts *opts);
1156 :
1157 : /**
1158 : * Get iobuf options.
1159 : *
1160 : * \param opts Output parameter for options.
1161 : * \param opts_size sizeof(*opts)
1162 : */
1163 : void spdk_iobuf_get_opts(struct spdk_iobuf_opts *opts, size_t opts_size);
1164 :
1165 : /**
1166 : * Register a module as an iobuf pool user. Only registered users can request buffers from the
1167 : * iobuf pool.
1168 : *
1169 : * \name Name of the module.
1170 : *
1171 : * \return 0 on success, negative errno otherwise.
1172 : */
1173 : int spdk_iobuf_register_module(const char *name);
1174 :
1175 : /**
1176 : * Unregister an iobuf pool user from a module.
1177 : *
1178 : * \name Name of the module.
1179 : *
1180 : * \return 0 on success, negative errno otherwise.
1181 : */
1182 : int spdk_iobuf_unregister_module(const char *name);
1183 :
1184 : /**
1185 : * Initialize an iobuf channel.
1186 : *
1187 : * \param ch iobuf channel to initialize.
1188 : * \param name Name of the module registered via `spdk_iobuf_register_module()`.
1189 : * \param small_cache_size Number of small buffers to be cached by this channel.
1190 : * \param large_cache_size Number of large buffers to be cached by this channel.
1191 : *
1192 : * \return 0 on success, negative errno otherwise.
1193 : */
1194 : int spdk_iobuf_channel_init(struct spdk_iobuf_channel *ch, const char *name,
1195 : uint32_t small_cache_size, uint32_t large_cache_size);
1196 :
1197 : /**
1198 : * Release resources tied to an iobuf channel.
1199 : *
1200 : * \param ch iobuf channel.
1201 : */
1202 : void spdk_iobuf_channel_fini(struct spdk_iobuf_channel *ch);
1203 :
1204 : typedef int (*spdk_iobuf_for_each_entry_fn)(struct spdk_iobuf_channel *ch,
1205 : struct spdk_iobuf_entry *entry, void *ctx);
1206 :
1207 : /**
1208 : * Iterate over all entries on a given channel and execute a callback on those that were requested.
1209 : * The iteration is stopped if the callback returns non-zero status.
1210 : *
1211 : * \param ch iobuf channel to iterate over.
1212 : * \param cb_fn Callback to execute on each entry on the channel that was requested.
1213 : * \param cb_ctx Argument passed to `cb_fn`.
1214 : *
1215 : * \return status of the last callback.
1216 : */
1217 : int spdk_iobuf_for_each_entry(struct spdk_iobuf_channel *ch,
1218 : spdk_iobuf_for_each_entry_fn cb_fn, void *cb_ctx);
1219 :
1220 : /**
1221 : * Abort an outstanding request waiting for a buffer.
1222 : *
1223 : * \param ch iobuf channel on which the entry is waiting.
1224 : * \param entry Entry to remove from the wait queue.
1225 : * \param len Length of the requested buffer (must be the exact same value as specified in
1226 : * `spdk_iobuf_get()`.
1227 : */
1228 : void spdk_iobuf_entry_abort(struct spdk_iobuf_channel *ch, struct spdk_iobuf_entry *entry,
1229 : uint64_t len);
1230 :
1231 : /**
1232 : * Get a buffer from the iobuf pool. If no buffers are available and entry with cb_fn provided
1233 : * then the request is queued until a buffer becomes available.
1234 : *
1235 : * \param ch iobuf channel.
1236 : * \param len Length of the buffer to retrieve. The user is responsible for making sure the length
1237 : * doesn't exceed large_bufsize.
1238 : * \param entry Wait queue entry (optional).
1239 : * \param cb_fn Callback to be executed once a buffer becomes available. If a buffer is available
1240 : * immediately, it is NOT executed. Mandatory only if entry provided.
1241 : *
1242 : * \return pointer to a buffer or NULL if no buffers are currently available.
1243 : */
1244 : void *spdk_iobuf_get(struct spdk_iobuf_channel *ch, uint64_t len, struct spdk_iobuf_entry *entry,
1245 : spdk_iobuf_get_cb cb_fn);
1246 :
1247 : /**
1248 : * Release a buffer back to the iobuf pool. If there are outstanding requests waiting for a buffer,
1249 : * this buffer will be passed to one of them.
1250 : *
1251 : * \param ch iobuf channel.
1252 : * \param buf Buffer to release
1253 : * \param len Length of the buffer (must be the exact same value as specified in `spdk_iobuf_get()`).
1254 : */
1255 : void spdk_iobuf_put(struct spdk_iobuf_channel *ch, void *buf, uint64_t len);
1256 :
1257 : typedef void (*spdk_iobuf_get_stats_cb)(struct spdk_iobuf_module_stats *modules,
1258 : uint32_t num_modules, void *cb_arg);
1259 :
1260 : /**
1261 : * Get iobuf statistics.
1262 : *
1263 : * \param cb_fn Callback to be executed once stats are gathered.
1264 : * \param cb_arg Argument passed to the callback function.
1265 : *
1266 : * \return 0 on success, negative errno otherwise.
1267 : */
1268 : int spdk_iobuf_get_stats(spdk_iobuf_get_stats_cb cb_fn, void *cb_arg);
1269 :
1270 : #ifdef __cplusplus
1271 : }
1272 : #endif
1273 :
1274 : #endif /* SPDK_THREAD_H_ */
|