gdb: warning: Couldn't determine a path for the index cache directory. warning: Can't open file anon_inode:[vfio-device] which was expanded to anon_inode:[vfio-device] during file-backed mapping note processing [New LWP 357984] [New LWP 358007] [New LWP 358006] [Thread debugging using libthread_db enabled] Using host libthread_db library "/usr/lib64/libthread_db.so.1". Core was generated by `/var/jenkins/workspace/nvme-phy-autotest/spdk/build/examples/hotplug -i 0 -t 0'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f141765a884 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 [Current thread is 1 (Thread 0x7f1416c28a00 (LWP 357984))] Thread 3 (Thread 0x7f1416c006c0 (LWP 358006)): #0 0x00007f14176ded72 in epoll_wait () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f141819e85c in eal_intr_handle_interrupts (pfd=6, totalfds=4) at ../lib/eal/linux/eal_interrupts.c:1077 events = {{events = 1, data = {ptr = 0x4, fd = 4, u32 = 4, u64 = 4}}, {events = 32532, data = {ptr = 0x400000001, fd = 1, u32 = 1, u64 = 17179869185}}, {events = 0, data = {ptr = 0x100007f14, fd = 32532, u32 = 32532, u64 = 4294999828}}, {events = 4, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}}} nfds = 0 #2 0x00007f141819ea99 in eal_intr_thread_main (arg=0x0) at ../lib/eal/linux/eal_interrupts.c:1163 pipe_event = {events = 3, data = {ptr = 0x4, fd = 4, u32 = 4, u64 = 4}} src = 0x0 numfds = 4 pfd = 6 __func__ = "eal_intr_thread_main" #3 0x00007f141817ccd7 in control_thread_start (arg=0x6ddde0) at ../lib/eal/common/eal_common_thread.c:282 params = 0x6ddde0 start_arg = 0x0 start_routine = 0x7f141819e8cb #4 0x00007f1418195a2a in thread_start_wrapper (arg=0x7ffd53ece970) at ../lib/eal/unix/rte_thread.c:114 ctx = 0x7ffd53ece970 thread_func = 0x7f141817cc88 thread_args = 0x6ddde0 ret = 0 #5 0x00007f1417658947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #6 0x00007f14176de970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 2 (Thread 0x7f14162006c0 (LWP 358007)): #0 0x00007f14176e088b in recvmsg () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f141818bdb9 in read_msg (fd=9, m=0x7f14161fe9f0, s=0x7f14161fe980) at ../lib/eal/common/eal_common_proc.c:284 msglen = 0 iov = {iov_base = 0x7f14161fe9f0, iov_len = 332} msgh = {msg_name = 0x7f14161fe980, msg_namelen = 110, msg_iov = 0x7f14161fe940, msg_iovlen = 1, msg_control = 0x7f14161fe8d0, msg_controllen = 48, msg_flags = 0} control = '\000' cmsg = 0x0 buflen = 332 #2 0x00007f141818c306 in mp_handle (arg=0x0) at ../lib/eal/common/eal_common_proc.c:410 ret = 404567616 msg = {type = 0, msg = {name = '\000' , len_param = 0, num_fds = 0, param = '\000' , "\001\000\000\000\320\352\037\026\024\177\000\000=\300\027\030\024\177\000\000\000\000\000\000\000\000\000\000\210j\035\030\024\177", '\000' , "\001\000\000\000\026\000\000\000\000\000\000\000\377\377\377\377\026\000\000\000\020\353\037\026\024\177\000\000\300\300\027\030\024\177\000\000\020\353\037\026\024\177\000\000\262\361\030\030\024\177\000\000\020\353\037\026\024\177\000\000h\277\027\030\377\377\377\377@\300\034\030"..., fds = {32532, 0, 0, 7200224, 0, 0, 0, 7200224}}} sa = {sun_family = 0, sun_path = '\000' } fd = 9 #3 0x00007f141817ccd7 in control_thread_start (arg=0x6ddde0) at ../lib/eal/common/eal_common_thread.c:282 params = 0x6ddde0 start_arg = 0x0 start_routine = 0x7f141818c2d7 #4 0x00007f1418195a2a in thread_start_wrapper (arg=0x7ffd53ecd980) at ../lib/eal/unix/rte_thread.c:114 ctx = 0x7ffd53ecd980 thread_func = 0x7f141817cc88 thread_args = 0x6ddde0 ret = 0 #5 0x00007f1417658947 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #6 0x00007f14176de970 in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 1 (Thread 0x7f1416c28a00 (LWP 357984)): #0 0x00007f141765a884 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f1417609afe in raise () from /usr/lib64/libc.so.6 No symbol table info available. #2 0x00007f14175f287f in abort () from /usr/lib64/libc.so.6 No symbol table info available. #3 0x00007f14177c97cf in __sanitizer::Abort() () from /usr/lib64/libubsan.so.1 No symbol table info available. #4 0x00007f14177d8cc1 in __sanitizer::Die() () from /usr/lib64/libubsan.so.1 No symbol table info available. #5 0x00007f14177b3426 in __ubsan::ScopedReport::~ScopedReport() () from /usr/lib64/libubsan.so.1 No symbol table info available. #6 0x00007f14177b6b79 in handleTypeMismatchImpl(__ubsan::TypeMismatchData*, unsigned long, __ubsan::ReportOptions) () from /usr/lib64/libubsan.so.1 No symbol table info available. #7 0x00007f14177b7e2f in __ubsan_handle_type_mismatch_v1 () from /usr/lib64/libubsan.so.1 No symbol table info available. #8 0x00007f141866b33b in _nvme_pcie_ctrlr_create_io_qpair (ctrlr=0x2000002fc080, qpair=0x200000600be0, qid=1) at nvme_pcie_common.c:535 group = 0x505050505050505 pqpair = 0x200000600b80 rc = 0 __func__ = "_nvme_pcie_ctrlr_create_io_qpair" #9 0x00007f141866b7c4 in nvme_pcie_ctrlr_connect_qpair (ctrlr=0x2000002fc080, qpair=0x200000600be0) at nvme_pcie_common.c:565 rc = 0 #10 0x00007f14186a1eda in nvme_transport_ctrlr_connect_qpair (ctrlr=0x2000002fc080, qpair=0x200000600be0) at nvme_transport.c:478 transport = 0x7f14188237a0 rc = 32532 __PRETTY_FUNCTION__ = "nvme_transport_ctrlr_connect_qpair" #11 0x00007f14186106c9 in spdk_nvme_ctrlr_connect_io_qpair (ctrlr=0x2000002fc080, qpair=0x200000600be0) at nvme_ctrlr.c:432 rc = 0 #12 0x00007f1418610e4c in spdk_nvme_ctrlr_alloc_io_qpair (ctrlr=0x2000002fc080, user_opts=0x0, opts_size=0) at nvme_ctrlr.c:504 qpair = 0x200000600be0 opts = {qprio = SPDK_NVME_QPRIO_URGENT, io_queue_size = 1024, io_queue_requests = 1024, {delay_cmd_submit = false, delay_pcie_doorbell = false}, reserved13 = "\000\000", sq = {vaddr = 0x0, paddr = 0, buffer_size = 0}, cq = {vaddr = 0x0, paddr = 0, buffer_size = 0}, create_only = false, async_mode = false, reserved66 = "\000\000\000\000\000"} rc = 32765 __func__ = "spdk_nvme_ctrlr_alloc_io_qpair" #13 0x000000000040306e in register_dev (ctrlr=0x2000002fc080) at hotplug.c:106 dev = 0x759e00 cdata = 0x2000002fc668 #14 0x0000000000404a1a in attach_cb (cb_ctx=0x0, trid=0x2000002fc0a8, ctrlr=0x2000002fc080, opts=0x2000002fd688) at hotplug.c:275 No locals. #15 0x00007f1418693ffc in nvme_ctrlr_poll_internal (ctrlr=0x2000002fc080, probe_ctx=0x759aa0) at nvme.c:743 rc = 0 __func__ = "nvme_ctrlr_poll_internal" #16 0x00007f141869bf98 in spdk_nvme_probe_poll_async (probe_ctx=0x759aa0) at nvme.c:1548 ctrlr = 0x2000002fc080 ctrlr_tmp = 0x0 #17 0x00007f1418694036 in nvme_init_controllers (probe_ctx=0x759aa0) at nvme.c:753 rc = -11 #18 0x00007f1418695b11 in spdk_nvme_probe (trid=0x7ffd53ecea30, cb_ctx=0x0, probe_cb=0x404866 , attach_cb=0x40493f , remove_cb=0x404a2c ) at nvme.c:921 trid_pcie = {trstring = "PCIE", '\000' , trtype = SPDK_NVME_TRANSPORT_PCIE, adrfam = 0, traddr = '\000' , trsvcid = '\000' , subnqn = '\000' , priority = 0} probe_ctx = 0x759aa0 __func__ = "spdk_nvme_probe" #19 0x0000000000404efa in io_loop () at hotplug.c:356 now = 4889672003938626 dev = 0x0 dev_tmp = 0x0 tsc_end = 18446744073709551615 next_stats_tsc = 4889665852807548 rc = 0 __func__ = "io_loop" #20 0x0000000000405e37 in main (argc=11, argv=0x7ffd53eceea8) at hotplug.c:593 rc = 0 opts = {name = 0x418519 "hotplug", core_mask = 0x418521 "0x1", lcore_map = 0x0, shm_id = 0, mem_channel = -1, main_core = -1, mem_size = -1, no_pci = false, hugepage_single_segments = false, unlink_hugepage = false, no_huge = false, num_pci_addr = 0, hugedir = 0x0, pci_blocked = 0x0, pci_allowed = 0x0, iova_mode = 0x0, base_virtaddr = 35184372088832, env_context = 0x0, vf_token = 0x0}