gdb: warning: Couldn't determine a path for the index cache directory. [New LWP 2222864] [New LWP 2222868] [New LWP 2222870] [New LWP 2222866] [New LWP 2222867] [New LWP 2222869] [Thread debugging using libthread_db enabled] Using host libthread_db library "/usr/lib64/libthread_db.so.1". Core was generated by `/var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/build/bin/nvmf_tgt -i 0 -e 0x'. Program terminated with signal SIGABRT, Aborted. #0 0x00007f9d25fe7834 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 [Current thread is 1 (Thread 0x7f9d24d4ba00 (LWP 2222864))] Thread 6 (Thread 0x7f9d22e006c0 (LWP 2222869)): #0 0x00007f9d26000dd8 in __memmove_sse2_unaligned_erms () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f9d26b54f6a in _sw_accel_copy_iovs (dst_iovs=0x7f9d141db010, dst_iovcnt=1, src_iovs=0x2000070dac68, src_iovcnt=1) at accel_sw.c:122 iter = {count = 2, {iters_compat = {{iov = 0x2000070dac68, iovcnt = 1, idx = 1, iov_len = 4096, iov_base = 0x200014a1b000 ""}, {iov = 0x7f9d141db010, iovcnt = 1, idx = 1, iov_len = 4096, iov_base = 0x2000076fe000 ""}}, iters = 0x7f9d22dfddb8}} src = 0x200014a1b000 dst = 0x2000076fe000 len = 4096 #2 0x00007f9d26b5a5bd in sw_accel_submit_tasks (ch=0x7f9d1413b470, accel_task=0x7f9d1400f400) at accel_sw.c:530 sw_ch = 0x7f9d1413b4d0 tmp = 0x0 rc = 0 __PRETTY_FUNCTION__ = "sw_accel_submit_tasks" #3 0x00007f9d26b2942e in accel_submit_task (accel_ch=0x7f9d1400f0c0, task=0x7f9d1400f400) at accel.c:350 module_ch = 0x7f9d1413b470 module = 0x7f9d26b7d0e0 rc = 0 #4 0x00007f9d26b40b35 in accel_process_sequence (seq=0x7f9d140af400) at accel.c:1867 accel_ch = 0x7f9d1400f0c0 task = 0x7f9d1400f400 state = ACCEL_SEQUENCE_STATE_INIT rc = 0 __PRETTY_FUNCTION__ = "accel_process_sequence" __func__ = "accel_process_sequence" #5 0x00007f9d26b439af in spdk_accel_sequence_finish (seq=0x7f9d140af400, cb_fn=0x7f9d28490b71 , cb_arg=0x2000070dac68) at accel.c:2136 task = 0x7f9d1400f400 next = 0x0 #6 0x00007f9d2849179c in bdev_malloc_readv (mdisk=0x1d4e850, ch=0x7f9d1400f060, task=0x2000070dac68, bdev_io=0x2000070da880) at bdev_malloc.c:355 len = 4096 offset = 4304896 res = 0 __func__ = "bdev_malloc_readv" #7 0x00007f9d28493644 in _bdev_malloc_submit_request (mch=0x7f9d141acaf0, bdev_io=0x2000070da880) at bdev_malloc.c:482 task = 0x2000070dac68 disk = 0x1d4e850 block_size = 512 rc = 772951 __PRETTY_FUNCTION__ = "_bdev_malloc_submit_request" #8 0x00007f9d28494048 in bdev_malloc_submit_request (ch=0x7f9d141aca90, bdev_io=0x2000070da880) at bdev_malloc.c:552 mch = 0x7f9d141acaf0 #9 0x00007f9d26c59434 in bdev_submit_request (bdev=0x1d4e850, ioch=0x7f9d141aca90, bdev_io=0x2000070da880) at bdev.c:1457 __PRETTY_FUNCTION__ = "bdev_submit_request" #10 0x00007f9d26c66622 in bdev_io_do_submit (bdev_ch=0x7f9d141ac9d0, bdev_io=0x2000070da880) at bdev.c:2783 bdev = 0x1d4e850 ch = 0x7f9d141aca90 shared_resource = 0x7f9d141ace00 __func__ = "bdev_io_do_submit" #11 0x00007f9d26c6e1f0 in _bdev_io_submit (ctx=0x2000070da880) at bdev.c:3505 bdev_io = 0x2000070da880 bdev = 0x1d4e850 bdev_ch = 0x7f9d141ac9d0 __func__ = "_bdev_io_submit" #12 0x00007f9d26c6fc01 in bdev_io_submit (bdev_io=0x2000070da880) at bdev.c:3618 ch = 0x7f9d141ac9d0 __PRETTY_FUNCTION__ = "bdev_io_submit" #13 0x00007f9d26c703b7 in _bdev_io_submit_ext (desc=0x20ae100, bdev_io=0x2000070da880) at bdev.c:3672 ch = 0x7f9d141ac9d0 needs_exec = false __PRETTY_FUNCTION__ = "_bdev_io_submit_ext" #14 0x00007f9d26c8542b in bdev_readv_blocks_with_md (desc=0x20ae100, ch=0x7f9d141ac970, iov=0x7f9d141db010, iovcnt=1, md_buf=0x0, offset_blocks=8408, num_blocks=8, domain=0x0, domain_ctx=0x0, seq=0x0, dif_check_flags=0, cb=0x7f9d27330df3 , cb_arg=0x7f9d141dafc0) at bdev.c:5504 bdev = 0x1d4e850 bdev_io = 0x2000070da880 channel = 0x7f9d141ac9d0 #15 0x00007f9d26c85d3e in spdk_bdev_readv_blocks_ext (desc=0x20ae100, ch=0x7f9d141ac970, iov=0x7f9d141db010, iovcnt=1, offset_blocks=8408, num_blocks=8, cb=0x7f9d27330df3 , cb_arg=0x7f9d141dafc0, opts=0x7f9d22dfe3d0) at bdev.c:5596 domain = 0x0 seq = 0x0 domain_ctx = 0x0 md = 0x0 dif_check_flags = 0 bdev = 0x1d4e850 #16 0x00007f9d27333d36 in nvmf_bdev_ctrlr_read_cmd (bdev=0x1d4e850, desc=0x20ae100, ch=0x7f9d141ac970, req=0x7f9d141dafc0) at ctrlr_bdev.c:318 opts = {size = 40, memory_domain = 0x0, memory_domain_ctx = 0x0, metadata = 0x0, accel_sequence = 0x0, dif_check_flags_exclude_mask = 0, nvme_cdw12 = {raw = 0, write = {reserved = 0, dtype = 0, reserved2 = 0}}, nvme_cdw13 = {raw = 0, write = {reserved = 0, dspec = 0}}} bdev_num_blocks = 131072 block_size = 512 cmd = 0x7f9d141db2f8 rsp = 0x7f9d141db2e8 start_lba = 8408 num_blocks = 8 rc = 16777216 __func__ = "nvmf_bdev_ctrlr_read_cmd" __PRETTY_FUNCTION__ = "nvmf_bdev_ctrlr_read_cmd" #17 0x00007f9d27308e65 in nvmf_ctrlr_process_io_cmd (req=0x7f9d141dafc0) at ctrlr.c:4443 nsid = 1 ns = 0x20ae030 bdev = 0x1d4e850 desc = 0x20ae100 ch = 0x7f9d141ac970 qpair = 0x1d486d0 group = 0x7f9d14000c00 ctrlr = 0x7f9d101d31d0 cmd = 0x7f9d141db2f8 response = 0x7f9d141db2e8 ns_info = 0x7f9d141a9f60 ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE __PRETTY_FUNCTION__ = "nvmf_ctrlr_process_io_cmd" __func__ = "nvmf_ctrlr_process_io_cmd" #18 0x00007f9d2733304b in nvmf_ctrlr_process_io_cmd_resubmit (arg=0x7f9d141dafc0) at ctrlr_bdev.c:240 req = 0x7f9d141dafc0 rc = 32669 #19 0x00007f9d26c644da in spdk_bdev_free_io (bdev_io=0x2000070da880) at bdev.c:2513 entry = 0x7f9d141db270 ch = 0x7f9d141acd20 __PRETTY_FUNCTION__ = "spdk_bdev_free_io" #20 0x00007f9d2733142e in nvmf_bdev_ctrlr_complete_cmd (bdev_io=0x2000070da880, success=true, cb_arg=0x7f9d141da840) at ctrlr_bdev.c:102 req = 0x7f9d141da840 response = 0x7f9d141dab68 sc = 0 sct = 0 cdw0 = 0 #21 0x00007f9d26c96b72 in _bdev_io_complete (ctx=0x2000070da880) at bdev.c:7335 bdev_io = 0x2000070da880 __PRETTY_FUNCTION__ = "_bdev_io_complete" #22 0x00007f9d26c97276 in bdev_io_complete (ctx=0x2000070da880) at bdev.c:7374 bdev_io = 0x2000070da880 bdev_ch = 0x7f9d141ac9d0 tsc = 3319799982886158 tsc_diff = 436815 __PRETTY_FUNCTION__ = #23 0x00007f9d26c987b6 in spdk_bdev_io_complete (bdev_io=0x2000070da880, status=SPDK_BDEV_IO_STATUS_SUCCESS) at bdev.c:7502 bdev = 0x1d4e850 bdev_ch = 0x7f9d141ac9d0 shared_resource = 0x7f9d141ace00 __func__ = "spdk_bdev_io_complete" __PRETTY_FUNCTION__ = "spdk_bdev_io_complete" #24 0x00007f9d2848fb9e in malloc_done (ref=0x2000070dac68, status=0) at bdev_malloc.c:199 task = 0x2000070dac68 bdev_io = 0x2000070da880 rc = 32669 __PRETTY_FUNCTION__ = "malloc_done" #25 0x00007f9d28490c43 in malloc_sequence_done (ctx=0x2000070dac68, status=0) at bdev_malloc.c:316 task = 0x2000070dac68 bdev_io = 0x2000070da880 #26 0x00007f9d26b3994a in accel_sequence_complete (seq=0x7f9d140af400) at accel.c:1428 cb_fn = 0x7f9d28490b71 cb_arg = 0x2000070dac68 status = 0 __func__ = "accel_sequence_complete" #27 0x00007f9d26b4115f in accel_process_sequence (seq=0x7f9d140af400) at accel.c:1897 accel_ch = 0x7f9d1400f0c0 task = 0x0 state = ACCEL_SEQUENCE_STATE_NEXT_TASK rc = 32669 __PRETTY_FUNCTION__ = "accel_process_sequence" __func__ = "accel_process_sequence" #28 0x00007f9d26b4199b in accel_sequence_task_cb (cb_arg=0x7f9d140af400, status=0) at accel.c:1964 seq = 0x7f9d140af400 task = 0x7f9d1400f400 __func__ = "accel_sequence_task_cb" __PRETTY_FUNCTION__ = "accel_sequence_task_cb" #29 0x00007f9d26b28cc3 in spdk_accel_task_complete (accel_task=0x7f9d1400f400, status=0) at accel.c:321 accel_ch = 0x7f9d1400f0c0 cb_fn = 0x7f9d1400ee70 cb_arg = 0x1f11120 #30 0x00007f9d26b5a1eb in accel_comp_poll (arg=0x7f9d1413b4d0) at accel_sw.c:506 sw_ch = 0x7f9d1413b4d0 tasks_to_complete = {stqh_first = 0x7f9d1400f500, stqh_last = 0x7f9d1400f510} accel_task = 0x7f9d1400f400 #31 0x00007f9d269dde33 in thread_execute_poller (thread=0x1d51400, poller=0x7f9d141fa950) at thread.c:963 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #32 0x00007f9d269dfc5b in thread_poll (thread=0x1d51400, max_msgs=0, now=3319799982866439) at thread.c:1089 poller_rc = 1 msg_count = 0 poller = 0x7f9d141fa950 tmp = 0x7f9d141acb20 critical_msg = 0x0 rc = 0 #33 0x00007f9d269e0948 in spdk_thread_poll (thread=0x1d51400, max_msgs=0, now=3319799982866439) at thread.c:1177 orig_thread = 0x0 rc = 0 #34 0x00007f9d26db2d8c in _reactor_run (reactor=0x1d3cec0) at reactor.c:918 thread = 0x1d51400 lw_thread = 0x1d51748 tmp = 0x0 now = 3319799982866439 rc = 1 #35 0x00007f9d26db3400 in reactor_run (arg=0x1d3cec0) at reactor.c:956 reactor = 0x1d3cec0 thread = 0x1 lw_thread = 0x0 tmp = 0x0 thread_name = "reactor_2\000\000\000\000\000\000\000\037\000\000\000\000\000\000\000\327\250s&\235\177\000" last_sched = 0 __func__ = "reactor_run" #36 0x00007f9d2671878a in eal_thread_loop (arg=) at ../lib/eal/common/eal_common_thread.c:211 f = fct_arg = lcore_id = 2 cpuset = "2", '\000' ret = #37 0x00007f9d25fe5897 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #38 0x00007f9d2606ca5c in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 5 (Thread 0x7f9d242006c0 (LWP 2222867)): #0 0x00007f9d2606ebbb in recvmsg () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f9d267351de in read_msg (s=0x7f9d241fe880, m=0x7f9d241fea60, fd=10) at ../lib/eal/common/eal_common_proc.c:284 msglen = control = '\000' cmsg = buflen = 332 iov = {iov_base = 0x7f9d241fea60, iov_len = 332} msgh = {msg_name = 0x7f9d241fe880, msg_namelen = 110, msg_iov = 0x7f9d241fe840, msg_iovlen = 1, msg_control = 0x7f9d241fe850, msg_controllen = 48, msg_flags = 0} msglen = iov = msgh = control = cmsg = buflen = retry = #2 mp_handle (arg=) at ../lib/eal/common/eal_common_proc.c:410 ret = msg = {type = 0, msg = {name = '\000' , len_param = 0, num_fds = 0, param = '\000' , "\f", '\000' , "\310vv&\235\177\000\000\377\377\377\377\000\000\000\000\002\000\000\000"..., fds = {32669, 0, 0, 30232656, 0, 645297864, 32669, 645091600}}} sa = {sun_family = 0, sun_path = '\000' } fd = 10 #3 0x00007f9d25fe5897 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #4 0x00007f9d2606ca5c in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 4 (Thread 0x7f9d24c006c0 (LWP 2222866)): #0 0x00007f9d2606ce62 in epoll_wait () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f9d2673ebc4 in eal_intr_handle_interrupts (totalfds=, pfd=) at ../lib/eal/linux/eal_interrupts.c:1077 events = 0x7f9d24bfeb80 nfds = events = nfds = #2 eal_intr_thread_main (arg=) at ../lib/eal/linux/eal_interrupts.c:1163 pipe_event = {events = 3, data = {ptr = 0x5, fd = 5, u32 = 5, u64 = 5}} src = numfds = pfd = __func__ = "eal_intr_thread_main" #3 0x00007f9d25fe5897 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #4 0x00007f9d2606ca5c in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 3 (Thread 0x7f9d224006c0 (LWP 2222870)): #0 0x00007f9d273e1cbf in nvmf_tcp_req_put (tqpair=0x1f10c60, tcp_req=0x7f9d101d46c0) at tcp.c:466 __PRETTY_FUNCTION__ = "nvmf_tcp_req_put" #1 0x00007f9d274082dc in nvmf_tcp_req_process (ttransport=0x1d60390, tcp_req=0x7f9d101d46c0) at tcp.c:3311 tqpair = 0x1f10c60 plen = 34267392 pdu = 0x1f10c60 prev_state = TCP_REQUEST_STATE_COMPLETED progress = false transport = 0x1d60390 group = 0x7f9d1000ee70 tgroup = 0x7f9d101d46c0 __PRETTY_FUNCTION__ = "nvmf_tcp_req_process" __func__ = "nvmf_tcp_req_process" #2 0x00007f9d273e23c3 in nvmf_tcp_request_free (cb_arg=0x7f9d101d46c0) at tcp.c:499 ttransport = 0x1d60390 tcp_req = 0x7f9d101d46c0 __PRETTY_FUNCTION__ = "nvmf_tcp_request_free" __func__ = "nvmf_tcp_request_free" #3 0x00007f9d273ea354 in _req_pdu_write_done (req=0x7f9d101d46c0, err=0) at tcp.c:1151 tcp_req = 0x7f9d101d46c0 pdu = 0x200007095000 tqpair = 0x1f10c60 __PRETTY_FUNCTION__ = "_req_pdu_write_done" #4 0x00007f9d27a3825d in spdk_sock_request_complete (sock=0x1f10b10, req=0x2000070950b8, err=0) at /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/include/spdk_internal/sock.h:187 closed = false rc = 0 __PRETTY_FUNCTION__ = "spdk_sock_request_complete" #5 0x00007f9d27a38a39 in spdk_sock_request_put (sock=0x1f10b10, req=0x2000070950b8, err=0) at /var/jenkins/workspace/nvmf-tcp-phy-autotest/spdk/include/spdk_internal/sock.h:208 __PRETTY_FUNCTION__ = "spdk_sock_request_put" #6 0x00007f9d27a446cf in _sock_check_zcopy (sock=0x1f10b10) at posix.c:1321 psock = 0x1f10b10 msgh = {msg_name = 0x0, msg_namelen = 0, msg_iov = 0x0, msg_iovlen = 0, msg_control = 0x7f9d223fe3e0, msg_controllen = 32, msg_flags = 8200} buf = " ", '\000' , "\v\000\000\000\000\000\000\000\005\000\000\000\201\006\000\000\201\006\000" rc = 0 serr = 0x7f9d223fe3f0 cm = 0x7f9d223fe3e0 idx = 1665 req = 0x2000070950b8 treq = 0x2000070954f0 found = true __func__ = "_sock_check_zcopy" #7 0x00007f9d27a4b72a in posix_sock_group_impl_poll (_group=0x7f9d1000efb0, max_events=32, socks=0x7f9d223fe680) at posix.c:2113 group = 0x7f9d1000efb0 sock = 0x1f10b10 tmp = 0x0 num_events = 1 i = 0 rc = 24 psock = 0x1f10b10 ptmp = 0x7f9d101d5ca8 events = {{events = 9, data = {ptr = 0x1f10b10, fd = 32574224, u32 = 32574224, u64 = 32574224}}, {events = 32669, data = {ptr = 0x7f9d223fe4f0, fd = 574612720, u32 = 574612720, u64 = 140312861205744}}, {events = 657666123, data = {ptr = 0x20d48ba000007f9d, fd = 32669, u32 = 32669, u64 = 2365669223587413917}}, {events = 32669, data = {ptr = 0x7f9d101d5980, fd = 270358912, u32 = 270358912, u64 = 140312556951936}}, {events = 657497292, data = {ptr = 0x101d598000000001, fd = 1, u32 = 1, u64 = 1161182685222141953}}, {events = 32669, data = {ptr = 0x7f9d223fe530, fd = 574612784, u32 = 574612784, u64 = 140312861205808}}, {events = 650527962, data = {ptr = 0x223fe56c00007f9d, fd = 32669, u32 = 32669, u64 = 2467943372841582493}}, {events = 32669, data = {ptr = 0x2000070d98c0, fd = 118331584, u32 = 118331584, u64 = 35184490420416}}, {events = 270191904, data = {ptr = 0x101d5c3000007f9d, fd = 32669, u32 = 32669, u64 = 1161185640159674269}}, {events = 32669, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}}, {events = 657657331, data = {ptr = 0x223fe5b000007f9d, fd = 32669, u32 = 32669, u64 = 2467943664899358621}}, {events = 32669, data = {ptr = 0x7f9d2733142e , fd = 657658926, u32 = 657658926, u64 = 140312944251950}}, {events = 4096, data = {ptr = 0x101d55c000000000, fd = 0, u32 = 0, u64 = 1161178562053537792}}, {events = 32669, data = {ptr = 0x7f01223fe5a0, fd = 574612896, u32 = 574612896, u64 = 139642846307744}}, {events = 118331584, data = {ptr = 0x223fe5a000002000, fd = 8192, u32 = 8192, u64 = 2467943596179857408}}, {events = 32669, data = {ptr = 0x70d83c8, fd = 118326216, u32 = 118326216, u64 = 118326216}}, {events = 0, data = {ptr = 0x101aca9000000000, fd = 0, u32 = 0, u64 = 1160462573825425408}}, {events = 32669, data = {ptr = 0x7f9d101d55c0, fd = 270357952, u32 = 270357952, u64 = 140312556950976}}, {events = 270358760, data = {ptr = 0x4ffc8000007f9d, fd = 32669, u32 = 32669, u64 = 22514149846187933}}, {events = 8192, data = {ptr = 0x2000070d98c0, fd = 118331584, u32 = 118331584, u64 = 35184490420416}}, {events = 574612912, data = {ptr = 0x101d55c000007f9d, fd = 32669, u32 = 32669, u64 = 1161178562053570461}}, {events = 32669, data = {ptr = 0x7f9d223fe5f0, fd = 574612976, u32 = 574612976, u64 = 140312861206000}}, {events = 650734450, data = {ptr = 0x101ace6000007f9d, fd = 32669, u32 = 32669, u64 = 1160466765713538973}}, {events = 32669, data = {ptr = 0x2000070d98c0, fd = 118331584, u32 = 118331584, u64 = 35184490420416}}, {events = 270357952, data = {ptr = 0x70d98c000007f9d, fd = 32669, u32 = 32669, u64 = 508230283363909533}}, {events = 8192, data = {ptr = 0x0, fd = 0, u32 = 0, u64 = 0}}, {events = 270357952, data = {ptr = 0x223fe65000007f9d, fd = 32669, u32 = 32669, u64 = 2467944352094125981}}, {events = 32669, data = {ptr = 0x7f9d26c97276 , fd = 650736246, u32 = 650736246, u64 = 140312937329270}}, {events = 574613680, data = {ptr = 0x70d98c000007f9d, fd = 32669, u32 = 32669, u64 = 508230283363909533}}, {events = 8192, data = {ptr = 0x2000070d98c0, fd = 118331584, u32 = 118331584, u64 = 35184490420416}}, {events = 270191056, data = {ptr = 0x2ad3a23100007f9d, fd = 32669, u32 = 32669, u64 = 3085988501016051613}}, {events = 772951, data = {ptr = 0x43608, fd = 275976, u32 = 275976, u64 = 275976}}} last_placement_id = -1 __PRETTY_FUNCTION__ = "posix_sock_group_impl_poll" #8 0x00007f9d26a846b7 in sock_group_impl_poll_count (group_impl=0x7f9d1000efb0, group=0x7f9d1000ef00, max_events=32) at sock.c:757 socks = {0x7f9d101ac9d0, 0x7f9d101ace00, 0x2000004ffc80, 0x1, 0x7f9d223fe6f0, 0x7f9d2848fb9e , 0x223fe970, 0x2000070d9ca8, 0x2000004ffc80, 0x7f9d223fe578, 0x2000070d9ca8, 0x2000070d98c0, 0x1000, 0x7f9d1000f0c0, 0x7f9d223fe720, 0x7f9d28490c43 , 0x1000f0c0, 0x2000070d9ca8, 0x2000070d9ca8, 0x2000070d98c0, 0x7f9d223fe770, 0x7f9d26b3994a , 0x7f9d1000f400, 0x7f9d100af400, 0x7f9d223fe790, 0x1000f0c0, 0x7f9d28490b71 , 0x2000070d9ca8, 0x0, 0x68d200, 0x7f9d223fe7c0, 0x7f9d26b4115f } num_events = 1 i = 1 __PRETTY_FUNCTION__ = "sock_group_impl_poll_count" #9 0x00007f9d26a84b0c in spdk_sock_group_poll_count (group=0x7f9d1000ef00, max_events=32) at sock.c:791 group_impl = 0x7f9d1000efb0 rc = 0 num_events = 0 __func__ = "spdk_sock_group_poll_count" #10 0x00007f9d26a84532 in spdk_sock_group_poll (group=0x7f9d1000ef00) at sock.c:742 No locals. #11 0x00007f9d27409b7a in nvmf_tcp_poll_group_poll (group=0x7f9d1000ee70) at tcp.c:3482 tgroup = 0x7f9d1000ee70 num_events = 1 rc = 0 rc2 = 0 tqpair = 0x7f9d1000f0c0 tqpair_tmp = 0x0 __func__ = "nvmf_tcp_poll_group_poll" #12 0x00007f9d273c79a1 in nvmf_transport_poll_group_poll (group=0x7f9d1000ee70) at transport.c:735 No locals. #13 0x00007f9d2737de10 in nvmf_poll_group_poll (ctx=0x7f9d10000c00) at nvmf.c:157 group = 0x7f9d10000c00 rc = 0 count = 0 tgroup = 0x7f9d1000ee70 #14 0x00007f9d269dde33 in thread_execute_poller (thread=0x1d51800, poller=0x7f9d10000cc0) at thread.c:963 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #15 0x00007f9d269dfc5b in thread_poll (thread=0x1d51800, max_msgs=0, now=3319799984905404) at thread.c:1089 poller_rc = 0 msg_count = 0 poller = 0x7f9d10000cc0 tmp = 0x0 critical_msg = 0x0 rc = 1 #16 0x00007f9d269e0948 in spdk_thread_poll (thread=0x1d51800, max_msgs=0, now=3319799984905404) at thread.c:1177 orig_thread = 0x0 rc = 0 #17 0x00007f9d26db2d8c in _reactor_run (reactor=0x1d3d180) at reactor.c:918 thread = 0x1d51800 lw_thread = 0x1d51b48 tmp = 0x0 now = 3319799984905404 rc = 1 #18 0x00007f9d26db3400 in reactor_run (arg=0x1d3d180) at reactor.c:956 reactor = 0x1d3d180 thread = 0x1 lw_thread = 0x0 tmp = 0x0 thread_name = "reactor_3\000\000\000\000\000\000\000#\000\000\000\000\000\000\000\327\250s&\235\177\000" last_sched = 0 __func__ = "reactor_run" #19 0x00007f9d2671878a in eal_thread_loop (arg=) at ../lib/eal/common/eal_common_thread.c:211 f = fct_arg = lcore_id = 3 cpuset = "3", '\000' ret = #20 0x00007f9d25fe5897 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #21 0x00007f9d2606ca5c in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 2 (Thread 0x7f9d238006c0 (LWP 2222868)): #0 sock_group_impl_poll_count (group_impl=0x7f9d1c00efb0, group=0x7f9d1c00ef00, max_events=32) at sock.c:757 socks = {0x0, 0x0, 0x200003affc80, 0x7f9d237fe970, 0x0, 0x200003affc80, 0x7f9d237fe970, 0x0, 0x200003affc80, 0x7f9d237fe578, 0x7f9d237fe580, 0x7f9d237fe588, 0x0, 0x0, 0x0, 0x0, 0x0, 0x20000402ec00, 0x7f9d237fe750, 0x7f9d269e14c5 , 0x0, 0x1d51800, 0x7f9d237fe770, 0x1, 0x7f9d2680dbec , 0x20000402ec00, 0x7f9d237fe7b0, 0x7f9d269e1d85 , 0x1, 0x7f9d101d46c0, 0x7f9d272d53f1 , 0x1d51800} num_events = 0 i = 0 __PRETTY_FUNCTION__ = "sock_group_impl_poll_count" #1 0x00007f9d26a84b0c in spdk_sock_group_poll_count (group=0x7f9d1c00ef00, max_events=32) at sock.c:791 group_impl = 0x7f9d1c00efb0 rc = 0 num_events = 0 __func__ = "spdk_sock_group_poll_count" #2 0x00007f9d26a84532 in spdk_sock_group_poll (group=0x7f9d1c00ef00) at sock.c:742 No locals. #3 0x00007f9d27409b7a in nvmf_tcp_poll_group_poll (group=0x7f9d1c00ee70) at tcp.c:3482 tgroup = 0x7f9d1c00ee70 num_events = 0 rc = 0 rc2 = 32669 tqpair = 0x0 tqpair_tmp = 0x7f9d101d46c0 __func__ = "nvmf_tcp_poll_group_poll" #4 0x00007f9d273c79a1 in nvmf_transport_poll_group_poll (group=0x7f9d1c00ee70) at transport.c:735 No locals. #5 0x00007f9d2737de10 in nvmf_poll_group_poll (ctx=0x7f9d1c000c00) at nvmf.c:157 group = 0x7f9d1c000c00 rc = 0 count = 0 tgroup = 0x7f9d1c00ee70 #6 0x00007f9d269dde33 in thread_execute_poller (thread=0x1d51000, poller=0x7f9d1c000cc0) at thread.c:963 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #7 0x00007f9d269dfc5b in thread_poll (thread=0x1d51000, max_msgs=0, now=3319799985043881) at thread.c:1089 poller_rc = 0 msg_count = 0 poller = 0x7f9d1c000cc0 tmp = 0x0 critical_msg = 0x0 rc = 0 #8 0x00007f9d269e0948 in spdk_thread_poll (thread=0x1d51000, max_msgs=0, now=3319799985043881) at thread.c:1177 orig_thread = 0x0 rc = 0 #9 0x00007f9d26db2d8c in _reactor_run (reactor=0x1d3cc00) at reactor.c:918 thread = 0x1d51000 lw_thread = 0x1d51348 tmp = 0x0 now = 3319799985043881 rc = 0 #10 0x00007f9d26db3400 in reactor_run (arg=0x1d3cc00) at reactor.c:956 reactor = 0x1d3cc00 thread = 0x1 lw_thread = 0x0 tmp = 0x0 thread_name = "reactor_1\000\000\000\000\000\000\000\033\000\000\000\000\000\000\000\327\250s&\235\177\000" last_sched = 0 __func__ = "reactor_run" #11 0x00007f9d2671878a in eal_thread_loop (arg=) at ../lib/eal/common/eal_common_thread.c:211 f = fct_arg = lcore_id = 1 cpuset = "1", '\000' ret = #12 0x00007f9d25fe5897 in start_thread () from /usr/lib64/libc.so.6 No symbol table info available. #13 0x00007f9d2606ca5c in clone3 () from /usr/lib64/libc.so.6 No symbol table info available. Thread 1 (Thread 0x7f9d24d4ba00 (LWP 2222864)): #0 0x00007f9d25fe7834 in __pthread_kill_implementation () from /usr/lib64/libc.so.6 No symbol table info available. #1 0x00007f9d25f958ee in raise () from /usr/lib64/libc.so.6 No symbol table info available. #2 0x00007f9d25f7d8ff in abort () from /usr/lib64/libc.so.6 No symbol table info available. #3 0x00007f9d25f7d81b in __assert_fail_base.cold () from /usr/lib64/libc.so.6 No symbol table info available. #4 0x00007f9d25f8dc57 in __assert_fail () from /usr/lib64/libc.so.6 No symbol table info available. #5 0x00007f9d27409750 in nvmf_tcp_req_complete (req=0x1f4f150) at tcp.c:3442 ttransport = 0x1d60390 tcp_req = 0x1f4f150 __func__ = "nvmf_tcp_req_complete" __PRETTY_FUNCTION__ = "nvmf_tcp_req_complete" #6 0x00007f9d273c7cad in nvmf_transport_req_complete (req=0x1f4f150) at transport.c:747 No locals. #7 0x00007f9d2730b855 in _nvmf_request_complete (ctx=0x1f4f150) at ctrlr.c:4588 req = 0x1f4f150 rsp = 0x1f4f478 qpair = 0x1f11120 sgroup = 0x1d522c8 ns_info = 0x1f4f848 is_aer = false nsid = 1 paused = 244 opcode = 127 '\177' __PRETTY_FUNCTION__ = "_nvmf_request_complete" __func__ = "_nvmf_request_complete" #8 0x00007f9d27308c84 in nvmf_ctrlr_process_io_cmd (req=0x1f4f510) at ctrlr.c:4433 fused_response = 0x1f4f478 nsid = 1 ns = 0x20ae030 bdev = 0x1d4e850 desc = 0x20ae100 ch = 0x1efc9d0 qpair = 0x1f11120 group = 0x1d52030 ctrlr = 0x7f9d141d31d0 cmd = 0x1f4f848 response = 0x1f4f838 ns_info = 0x1efdc00 ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE __PRETTY_FUNCTION__ = "nvmf_ctrlr_process_io_cmd" __func__ = "nvmf_ctrlr_process_io_cmd" #9 0x00007f9d2730f51f in spdk_nvmf_request_exec (req=0x1f4f510) at ctrlr.c:4805 qpair = 0x1f11120 status = SPDK_NVMF_REQUEST_EXEC_STATUS_COMPLETE #10 0x00007f9d27406167 in nvmf_tcp_req_process (ttransport=0x1d60390, tcp_req=0x1f4f510) at tcp.c:3202 tqpair = 0x1f11120 plen = 32575440 pdu = 0x1f11120 prev_state = TCP_REQUEST_STATE_READY_TO_EXECUTE progress = false transport = 0x1d60390 group = 0x1d60720 tgroup = 0x1f4f510 __PRETTY_FUNCTION__ = "nvmf_tcp_req_process" __func__ = "nvmf_tcp_req_process" #11 0x00007f9d273f4313 in nvmf_tcp_capsule_cmd_payload_handle (ttransport=0x1d60390, tqpair=0x1f11120, pdu=0x200003afe3c8) at tcp.c:1864 tcp_req = 0x1f4f510 capsule_cmd = 0x200003afe3c8 error_offset = 0 fes = 32764 rsp = 0x1f4f838 __PRETTY_FUNCTION__ = "nvmf_tcp_capsule_cmd_payload_handle" __func__ = "nvmf_tcp_capsule_cmd_payload_handle" #12 0x00007f9d273f7a16 in _nvmf_tcp_pdu_payload_handle (tqpair=0x1f11120, pdu=0x200003afe3c8) at tcp.c:2125 ttransport = 0x1d60390 __func__ = "_nvmf_tcp_pdu_payload_handle" #13 0x00007f9d273f8e21 in nvmf_tcp_pdu_payload_handle (tqpair=0x1f11120, pdu=0x200003afe3c8) at tcp.c:2196 rc = 0 __PRETTY_FUNCTION__ = "nvmf_tcp_pdu_payload_handle" __func__ = "nvmf_tcp_pdu_payload_handle" #14 0x00007f9d273fd418 in nvmf_tcp_sock_process (tqpair=0x1f11120) at tcp.c:2549 rc = 16 pdu = 0x200003afe3c8 prev_state = NVME_TCP_PDU_RECV_STATE_AWAIT_PDU_PAYLOAD data_len = 16 ttransport = 0x1d60390 __func__ = "nvmf_tcp_sock_process" __PRETTY_FUNCTION__ = "nvmf_tcp_sock_process" #15 0x00007f9d2740848d in nvmf_tcp_sock_cb (arg=0x1f11120, group=0x1cf22e0, sock=0x1f10fd0) at tcp.c:3334 tqpair = 0x1f11120 rc = 32669 __PRETTY_FUNCTION__ = "nvmf_tcp_sock_cb" #16 0x00007f9d26a849aa in sock_group_impl_poll_count (group_impl=0x1d60810, group=0x1cf22e0, max_events=32) at sock.c:765 sock = 0x1f10fd0 socks = {0x1f10fd0, 0xd00000000, 0x200003eff200, 0x7ffca50ba140, 0x0, 0x200003eff200, 0x7ffca50ba140, 0x0, 0x200003eff200, 0x7ffca50b9d48, 0x7ffca50b9d50, 0x7ffca50b9d58, 0x1cef820, 0x1f4f150, 0x1d60390, 0x1f4f150, 0x1cb2290, 0x0, 0x7ffca50b9f10, 0x7f9d273c7cad , 0x0, 0x1f11120, 0x0, 0x0, 0x7ffca50b9f70, 0x7f9d2730c31a <_nvmf_request_complete+9806>, 0x0, 0x1f4f150, 0x17f000000, 0x1d522c8, 0x1f4f150, 0x1f4f478} num_events = 1 i = 0 __PRETTY_FUNCTION__ = "sock_group_impl_poll_count" #17 0x00007f9d26a84b0c in spdk_sock_group_poll_count (group=0x1cf22e0, max_events=32) at sock.c:791 group_impl = 0x1d60810 rc = 0 num_events = 0 __func__ = "spdk_sock_group_poll_count" #18 0x00007f9d26a84532 in spdk_sock_group_poll (group=0x1cf22e0) at sock.c:742 No locals. #19 0x00007f9d27409b7a in nvmf_tcp_poll_group_poll (group=0x1d60720) at tcp.c:3482 tgroup = 0x1d60720 num_events = 0 rc = 0 rc2 = 0 tqpair = 0x0 tqpair_tmp = 0xffffffff00000000 __func__ = "nvmf_tcp_poll_group_poll" #20 0x00007f9d273c79a1 in nvmf_transport_poll_group_poll (group=0x1d60720) at transport.c:735 No locals. #21 0x00007f9d2737de10 in nvmf_poll_group_poll (ctx=0x1d52030) at nvmf.c:157 group = 0x1d52030 rc = 0 count = 0 tgroup = 0x1d60720 #22 0x00007f9d269dde33 in thread_execute_poller (thread=0x1d51c00, poller=0x1d520f0) at thread.c:963 rc = 0 __PRETTY_FUNCTION__ = "thread_execute_poller" __func__ = "thread_execute_poller" #23 0x00007f9d269dfc5b in thread_poll (thread=0x1d51c00, max_msgs=0, now=3319799983447338) at thread.c:1089 poller_rc = 0 msg_count = 0 poller = 0x1d520f0 tmp = 0x0 critical_msg = 0x0 rc = 0 #24 0x00007f9d269e0948 in spdk_thread_poll (thread=0x1d51c00, max_msgs=0, now=3319799983447338) at thread.c:1177 orig_thread = 0x0 rc = 0 #25 0x00007f9d26db2d8c in _reactor_run (reactor=0x1d3c940) at reactor.c:918 thread = 0x1d51c00 lw_thread = 0x1d51f48 tmp = 0x0 now = 3319799983447338 rc = 0 #26 0x00007f9d26db3400 in reactor_run (arg=0x1d3c940) at reactor.c:956 reactor = 0x1d3c940 thread = 0x7f9d267235a8 lw_thread = 0x7f9d269e14c5 tmp = 0x8 thread_name = "reactor_0\000r&\235\177\000\000\001\000\000\000\000\000\000\000\347\313\260'\000\000\000" last_sched = 0 __func__ = "reactor_run" #27 0x00007f9d26db3f5d in spdk_reactors_start () at reactor.c:1072 reactor = 0x1d3c940 i = 4294967295 current_core = 0 rc = 0 __func__ = "spdk_reactors_start" __PRETTY_FUNCTION__ = "spdk_reactors_start" #28 0x00007f9d26da7ee4 in spdk_app_start (opts_user=0x7ffca50ba5f0, start_fn=0x4023e1 , arg1=0x0) at app.c:981 rc = 0 tty = 0x0 tmp_cpumask = {str = '\000' , cpus = "\001", '\000' } g_env_was_setup = false opts_local = {name = 0x42d01d "nvmf", json_config_file = 0x0, json_config_ignore_errors = false, reserved17 = "\000\000\000\000\000\000", rpc_addr = 0x7f9d26dc023c "/var/tmp/spdk.sock", reactor_mask = 0x7ffca50bbd0b "0xF", tpoint_group_mask = 0x7ffca50bbd01 "0xFFFF", shm_id = 0, reserved52 = "\000\000\000", shutdown_cb = 0x0, enable_coredump = true, reserved65 = "\000\000", mem_channel = -1, main_core = -1, mem_size = -1, no_pci = false, hugepage_single_segments = false, unlink_hugepage = false, no_huge = false, reserved84 = "\000\000\000", hugedir = 0x0, print_level = SPDK_LOG_INFO, reserved100 = "\000\000\000", num_pci_addr = 0, pci_blocked = 0x0, pci_allowed = 0x0, iova_mode = 0x0, delay_subsystem_init = true, reserved137 = "\000\000\000\000\000\000", num_entries = 32768, env_context = 0x0, log = 0x0, base_virtaddr = 35184372088832, opts_size = 253, disable_signal_handlers = false, interrupt_mode = false, reserved186 = "\000\000\000\000\000", msg_mempool_size = 262143, rpc_allowlist = 0x0, vf_token = 0x0, lcore_map = 0x0, rpc_log_level = SPDK_LOG_DISABLED, rpc_log_file = 0x0, json_data = 0x0, json_data_size = 0, disable_cpumask_locks = false} opts = 0x7ffca50ba310 i = 128 core = 4294967295 __func__ = "spdk_app_start" #29 0x0000000000402556 in main (argc=8, argv=0x7ffca50ba818) at nvmf_main.c:47 rc = 1 opts = {name = 0x42d01d "nvmf", json_config_file = 0x0, json_config_ignore_errors = false, reserved17 = "\000\000\000\000\000\000", rpc_addr = 0x7f9d26dc023c "/var/tmp/spdk.sock", reactor_mask = 0x7ffca50bbd0b "0xF", tpoint_group_mask = 0x7ffca50bbd01 "0xFFFF", shm_id = 0, reserved52 = "\000\000\000", shutdown_cb = 0x0, enable_coredump = true, reserved65 = "\000\000", mem_channel = -1, main_core = -1, mem_size = -1, no_pci = false, hugepage_single_segments = false, unlink_hugepage = false, no_huge = false, reserved84 = "\000\000\000", hugedir = 0x0, print_level = SPDK_LOG_INFO, reserved100 = "\000\000\000", num_pci_addr = 0, pci_blocked = 0x0, pci_allowed = 0x0, iova_mode = 0x0, delay_subsystem_init = true, reserved137 = "\000\000\000\000\000\000", num_entries = 32768, env_context = 0x0, log = 0x0, base_virtaddr = 35184372088832, opts_size = 253, disable_signal_handlers = false, interrupt_mode = false, reserved186 = "\000\000\000\000\000", msg_mempool_size = 0, rpc_allowlist = 0x0, vf_token = 0x0, lcore_map = 0x0, rpc_log_level = SPDK_LOG_DISABLED, rpc_log_file = 0x0, json_data = 0x0, json_data_size = 0, disable_cpumask_locks = false}