Thread 137 (Thread 0x7ffec11ff700 (LWP 3928)): #0 0x00007ffff6915e9d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000000441648 in sapp_usleep (usec=usec@entry=10000) at /home/yangwei/platform/sapp/src/dealpkt/plug_support.c:2144 #2 0x00000000004261aa in cycle_pkt_dump_udp_socket_daemon_thread (arg=) at /home/yangwei/platform/sapp/src/packet_io/cycle_pkt_dump_through_write_offset.c:1054 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 136 (Thread 0x7ffec09fe700 (LWP 3929)): #0 0x00007ffff6915e9d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000000441648 in sapp_usleep (usec=usec@entry=100) at /home/yangwei/platform/sapp/src/dealpkt/plug_support.c:2144 #2 0x0000000000466ffa in gdev_keepalive_log_thread (arg=) at /home/yangwei/platform/sapp/src/extensions/inline_keepalive.cpp:529 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 135 (Thread 0x7ffebdb31700 (LWP 3930)): #0 0x00007ffff59d30e3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007ffebecb54c4 in eal_intr_thread_main () from /opt/tsg/mrzcpd/lib/libmarsio.so #2 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 134 (Thread 0x7ffebcdff700 (LWP 3931)): #0 0x00007ffff6915bad in recvmsg () from /lib64/libpthread.so.0 #1 0x00007ffebecab162 in mp_handle () from /opt/tsg/mrzcpd/lib/libmarsio.so #2 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 133 (Thread 0x7ffebbfff700 (LWP 3932)): #0 0x00007ffff59d30e3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007ffebf5fb84e in __epoll_event_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #2 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 132 (Thread 0x7ffebb7fe700 (LWP 3933)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffebac14828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffebac14828) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ffebb7fdcb8, arena=arena@entry=0x7ffebac01040, cache_bin=cache_bin@entry=0x7ffebb7fe078, cache_bin_info=, binind=binind@entry=4, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ffebb7fdcb8, arena=arena@entry=0x7ffebac01040, tcache=tcache@entry=0x7ffebb7fe010, cache_bin=cache_bin@entry=0x7ffebb7fe078, binind=binind@entry=4, tcache_success=tcache_success@entry=0x7ffebb7f49c0) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=4, size=, tcache=0x7ffebb7fe010, arena=0x7ffebac01040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ffebb7fe010, zero=false, ind=4, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffebb7fe010, zero=false, ind=4, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=4, usize=64, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x00007ffebf5f818f in cJSON_New_Item () from /opt/tsg/mrzcpd/lib/libmarsio.so #16 0x00007ffebf5fa5bf in cJSON_CreateNumber () from /opt/tsg/mrzcpd/lib/libmarsio.so #17 0x00007ffebebeb7a2 in __create_uint64_array () from /opt/tsg/mrzcpd/lib/libmarsio.so #18 0x00007ffebebebe94 in __create_vdev_stats () from /opt/tsg/mrzcpd/lib/libmarsio.so #19 0x00007ffebebec791 in mrapp_monit_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #20 0x00007ffebebda1c0 in mrapp_ctrlplane_thread () from /opt/tsg/mrzcpd/lib/libmarsio.so #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 131 (Thread 0x7ffeba9ff700 (LWP 3934)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf7a03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf7a03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffdf7a03a28, tsdn=0x7ffeba9fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffeba9fecb8, pac=pac@entry=0x7ffdf7a039f0, ehooks=ehooks@entry=0x7ffdf7a000c0, ecache=ecache@entry=0x7ffdf7a03a28, expand_edata=expand_edata@entry=0x7ff70cf57800, size=size@entry=4096, alignment=4096, zero=false, commit=0x7ffeba9fd0bf, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffeba9fecb8, pac=pac@entry=0x7ffdf7a039f0, ehooks=ehooks@entry=0x7ffdf7a000c0, ecache=ecache@entry=0x7ffdf7a03a28, expand_edata=expand_edata@entry=0x7ff70cf57800, size=size@entry=4096, alignment=4096, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e87a9 in pac_expand_impl (tsdn=0x7ffeba9fecb8, self=0x7ffdf7a039f0, edata=0x7ff70cf57800, old_size=, new_size=, zero=, deferred_work_generated=0x7ffeba9fd1bf) at src/pac.c:202 #9 0x00000000004e7a6c in pai_expand (deferred_work_generated=0x7ffeba9fd1bf, zero=false, new_size=24576, old_size=20480, edata=0x7ff70cf57800, self=, tsdn=0x7ffeba9fecb8) at include/jemalloc/internal/pai.h:57 #10 je_pa_expand (tsdn=tsdn@entry=0x7ffeba9fecb8, shard=shard@entry=0x7ffdf7a039d8, edata=edata@entry=0x7ff70cf57800, old_size=20480, new_size=24576, szind=37, zero=false, deferred_work_generated=0x7ffeba9fd1bf) at src/pa.c:169 #11 0x00000000004e52e0 in large_ralloc_no_move_expand (tsdn=0x7ffeba9fecb8, edata=0x7ff70cf57800, usize=usize@entry=20480, zero=zero@entry=false) at src/large.c:94 #12 0x00000000004e5acf in je_large_ralloc_no_move (tsdn=tsdn@entry=0x7ffeba9fecb8, edata=edata@entry=0x7ff70cf57800, usize_min=usize_min@entry=20480, usize_max=usize_max@entry=20480, zero=zero@entry=false) at src/large.c:142 #13 0x00000000004e5fb7 in je_large_ralloc (tsdn=tsdn@entry=0x7ffeba9fecb8, arena=arena@entry=0x0, ptr=ptr@entry=0x7ff33fc2d5c0, usize=usize@entry=20480, alignment=alignment@entry=0, zero=zero@entry=false, tcache=0x7ffeba9ff010, hook_args=0x7ffeba9fd730) at src/large.c:197 #14 0x000000000049059e in je_arena_ralloc (tsdn=tsdn@entry=0x7ffeba9fecb8, arena=arena@entry=0x0, ptr=ptr@entry=0x7ff33fc2d5c0, oldsize=oldsize@entry=16384, size=size@entry=18432, alignment=alignment@entry=0, zero=false, tcache=0x7ffeba9ff010, hook_args=0x7ffeba9fd730) at src/arena.c:1480 #15 0x0000000000477e85 in iralloct (hook_args=0x7ffeba9fd730, arena=0x0, tcache=0x7ffeba9ff010, zero=, alignment=0, size=18432, oldsize=16384, ptr=0x7ff33fc2d5c0, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:194 #16 do_rallocx (ptr=ptr@entry=0x7ff33fc2d5c0, size=size@entry=18432, flags=flags@entry=0, is_realloc=is_realloc@entry=true) at src/jemalloc.c:3537 #17 0x000000000047b122 in realloc (ptr=0x7ff33fc2d5c0, size=18432) at src/jemalloc.c:3627 #18 0x00007ffff738d757 in prometheus_get_instance_metric_playload(fieldstat_instance*, char**, int*, int) () from /opt/tsg/framework/lib/libfieldstat3.so #19 0x00007ffff738f489 in prometheus_endpoint_instance_output(http_request_s*) () from /opt/tsg/framework/lib/libfieldstat3.so #20 0x00007ffff738f7f9 in hs_read_and_process_request(http_request_s*) () from /opt/tsg/framework/lib/libfieldstat3.so #21 0x00007ffff738f8b3 in http_session(http_request_s*) () from /opt/tsg/framework/lib/libfieldstat3.so #22 0x00007ffff738f990 in hs_accept_connections(http_server_s*) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff738edbb in http_server_listen_addr () from /opt/tsg/framework/lib/libfieldstat3.so #24 0x00007ffff738eddd in prometheus_endpoint_listen_thread_entry(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 130 (Thread 0x7ffeb6121700 (LWP 3935)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffff361dda8, mtx=mtx@entry=0x7ffff361dd80, tspec=tspec@entry=0x7ffeb6120120) at tinycthread_extra.c:106 #3 0x00007ffeb7086452 in rd_kafka_q_serve (rkq=0x7ffff361dd80, timeout_ms=, max_cnt=max_cnt@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_CALLBACK, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:482 #4 0x00007ffeb7045eac in rd_kafka_thread_main (arg=arg@entry=0x7ffff294f000) at rdkafka.c:2058 #5 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #6 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 129 (Thread 0x7ffeb5920700 (LWP 3936)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffff361de68, mtx=mtx@entry=0x7ffff361de40, tspec=tspec@entry=0x7ffeb591edc0) at tinycthread_extra.c:106 #3 0x00007ffeb7085e99 in rd_kafka_q_pop_serve (rkq=0x7ffff361de40, timeout_us=, version=version@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_RETURN, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:412 #4 0x00007ffeb7085f30 in rd_kafka_q_pop (rkq=, timeout_us=, version=version@entry=0) at rdkafka_queue.c:436 #5 0x00007ffeb706798f in rd_kafka_broker_ops_serve (rkb=rkb@entry=0x7fffbc1fd000, timeout_us=) at rdkafka_broker.c:3377 #6 0x00007ffeb7067a83 in rd_kafka_broker_ops_io_serve (rkb=rkb@entry=0x7fffbc1fd000, abs_timeout=, abs_timeout@entry=9747680554758) at rdkafka_broker.c:3433 #7 0x00007ffeb7069637 in rd_kafka_broker_internal_serve (abs_timeout=9747680554758, rkb=0x7fffbc1fd000) at rdkafka_broker.c:3610 #8 rd_kafka_broker_serve (rkb=rkb@entry=0x7fffbc1fd000, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5159 #9 0x00007ffeb7069c43 in rd_kafka_broker_thread_main (arg=arg@entry=0x7fffbc1fd000) at rdkafka_broker.c:5297 #10 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #11 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 128 (Thread 0x7ffeb47ff700 (LWP 3937)): #0 0x00007ffff59c7ddd in poll () from /lib64/libc.so.6 #1 0x00007ffeb707f38e in rd_kafka_transport_poll (rktrans=rktrans@entry=0x7ffeb3a0b000, tmout=tmout@entry=1000) at rdkafka_transport.c:977 #2 0x00007ffeb707f424 in rd_kafka_transport_io_serve (rktrans=0x7ffeb3a0b000, timeout_ms=1000) at rdkafka_transport.c:809 #3 0x00007ffeb7067a71 in rd_kafka_broker_ops_io_serve (rkb=rkb@entry=0x7fffbc1fdc00, abs_timeout=) at rdkafka_broker.c:3425 #4 0x00007ffeb7069320 in rd_kafka_broker_producer_serve (abs_timeout=9747680593866, rkb=0x7fffbc1fdc00) at rdkafka_broker.c:4032 #5 rd_kafka_broker_serve (rkb=rkb@entry=0x7fffbc1fdc00, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5164 #6 0x00007ffeb7069b65 in rd_kafka_broker_thread_main (arg=arg@entry=0x7fffbc1fdc00) at rdkafka_broker.c:5328 #7 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #8 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #9 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 127 (Thread 0x7ffdf77ff700 (LWP 3939)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf6c03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf6c03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffdf6c03a28, tsdn=0x7ffdf77fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffdf77fecb8, pac=pac@entry=0x7ffdf6c039f0, ehooks=ehooks@entry=0x7ffdf6c000c0, ecache=ecache@entry=0x7ffdf6c03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=32768, alignment=64, zero=false, commit=0x7ffdf77f936f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffdf77fecb8, pac=pac@entry=0x7ffdf6c039f0, ehooks=ehooks@entry=0x7ffdf6c000c0, ecache=ecache@entry=0x7ffdf6c03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=32768, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffdf77fecb8, pac=pac@entry=0x7ffdf6c039f0, ehooks=ehooks@entry=0x7ffdf6c000c0, size=size@entry=32768, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffdf77fecb8, self=0x7ffdf6c039f0, size=32768, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffdf77f94f7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffdf77f94f7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=32768, self=0x7ffdf6c039f0, tsdn=0x7ffdf77fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffdf77fecb8, shard=shard@entry=0x7ffdf6c039d8, size=size@entry=32768, alignment=alignment@entry=64, slab=slab@entry=false, szind=39, zero=false, guarded=false, deferred_work_generated=0x7ffdf77f94f7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffdf77fecb8, arena=arena@entry=0x7ffdf6c01040, usize=usize@entry=28672, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffdf77fecb8, arena=arena@entry=0x7ffdf6c01040, usize=28672, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffdf77fecb8, arena=arena@entry=0x7ffdf6c01040, usize=, zero=zero@entry=false) at src/large.c:17 #15 0x000000000049139f in tcache_alloc_large (slow_path=true, zero=, binind=, size=28672, tcache=0x7ffdf77ff010, arena=0x7ffdf6c01040, tsd=0x7ffdf77fecb8) at include/jemalloc/internal/tsd.h:283 #16 arena_malloc (slow_path=true, tcache=0x7ffdf77ff010, zero=, ind=, size=28672, arena=, tsdn=0x7ffdf77fecb8) at include/jemalloc/internal/arena_inlines_b.h:155 #17 arena_ralloc_move_helper (tcache=0x7ffdf77ff010, zero=, alignment=57343, usize=28672, arena=, tsdn=0x7ffdf77fecb8) at src/arena.c:1446 #18 je_arena_ralloc (tsdn=tsdn@entry=0x7ffdf77fecb8, arena=arena@entry=0x0, ptr=ptr@entry=0x7ff7c0e62000, oldsize=oldsize@entry=1536, size=size@entry=26152, alignment=alignment@entry=0, zero=false, tcache=0x7ffdf77ff010, hook_args=0x7ffdf77f9880) at src/arena.c:1488 #19 0x0000000000477e85 in iralloct (hook_args=0x7ffdf77f9880, arena=0x0, tcache=0x7ffdf77ff010, zero=, alignment=0, size=26152, oldsize=1536, ptr=0x7ff7c0e62000, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:194 #20 do_rallocx (ptr=ptr@entry=0x7ff7c0e62000, size=size@entry=26152, flags=flags@entry=0, is_realloc=is_realloc@entry=true) at src/jemalloc.c:3537 #21 0x000000000047b122 in realloc (ptr=0x7ff7c0e62000, size=26152) at src/jemalloc.c:3627 #22 0x00007ffeb65f1600 in sdsMakeRoomFor () from /opt/tsg/framework/lib/libmaat4.so.4 #23 0x00007ffeb65f1c3c in sdscatlen () from /opt/tsg/framework/lib/libmaat4.so.4 #24 0x00007ffeb65f0554 in redisReaderFeed () from /opt/tsg/framework/lib/libmaat4.so.4 #25 0x00007ffeb65ee2eb in redisBufferRead () from /opt/tsg/framework/lib/libmaat4.so.4 #26 0x00007ffeb65ee6a2 in redisGetReply () from /opt/tsg/framework/lib/libmaat4.so.4 #27 0x00007ffeb65ee6d7 in __redisBlockForReply () from /opt/tsg/framework/lib/libmaat4.so.4 #28 0x00007ffeb65ee9c7 in redisCommand () from /opt/tsg/framework/lib/libmaat4.so.4 #29 0x00007ffeb65cf773 in get_inc_key_list () from /opt/tsg/framework/lib/libmaat4.so.4 #30 0x00007ffeb65d06da in maat_get_rm_key_list () from /opt/tsg/framework/lib/libmaat4.so.4 #31 0x00007ffeb65d2a3f in redis_monitor_traverse () from /opt/tsg/framework/lib/libmaat4.so.4 #32 0x00007ffeb65cac87 in rule_monitor_loop () from /opt/tsg/framework/lib/libmaat4.so.4 #33 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #34 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 126 (Thread 0x7ffdf69ff700 (LWP 3940)): #0 0x00007ffff59d30e3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007ffebf5fb84e in __epoll_event_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #2 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 125 (Thread 0x7ffdf61fe700 (LWP 3941)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf5614828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf5614828) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ffdf61fdcb8, arena=arena@entry=0x7ffdf5601040, cache_bin=cache_bin@entry=0x7ffdf61fe078, cache_bin_info=, binind=binind@entry=4, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ffdf61fdcb8, arena=arena@entry=0x7ffdf5601040, tcache=tcache@entry=0x7ffdf61fe010, cache_bin=cache_bin@entry=0x7ffdf61fe078, binind=binind@entry=4, tcache_success=tcache_success@entry=0x7ffdf61f49c0) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=4, size=, tcache=0x7ffdf61fe010, arena=0x7ffdf5601040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ffdf61fe010, zero=false, ind=4, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffdf61fe010, zero=false, ind=4, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=4, usize=64, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x00007ffebf5f818f in cJSON_New_Item () from /opt/tsg/mrzcpd/lib/libmarsio.so #16 0x00007ffebf5fa5bf in cJSON_CreateNumber () from /opt/tsg/mrzcpd/lib/libmarsio.so #17 0x00007ffebebeb7a2 in __create_uint64_array () from /opt/tsg/mrzcpd/lib/libmarsio.so #18 0x00007ffebebebf5a in __create_vdev_stats () from /opt/tsg/mrzcpd/lib/libmarsio.so #19 0x00007ffebebec791 in mrapp_monit_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #20 0x00007ffebebda1c0 in mrapp_ctrlplane_thread () from /opt/tsg/mrzcpd/lib/libmarsio.so #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 124 (Thread 0x7ffb489ff700 (LWP 3946)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb47e03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb47e03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffb47e03a28, tsdn=0x7ffb489fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffb489fecb8, pac=pac@entry=0x7ffb47e039f0, ehooks=ehooks@entry=0x7ffb47e000c0, ecache=ecache@entry=0x7ffb47e03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=true, commit=0x7ffb489fd5ff, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffb489fecb8, pac=pac@entry=0x7ffb47e039f0, ehooks=ehooks@entry=0x7ffb47e000c0, ecache=ecache@entry=0x7ffb47e03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffb489fecb8, pac=pac@entry=0x7ffb47e039f0, ehooks=ehooks@entry=0x7ffb47e000c0, size=size@entry=462848, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffb489fecb8, self=0x7ffb47e039f0, size=462848, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffb489fd787) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffb489fd787, frequent_reuse=false, guarded=false, zero=true, alignment=, size=462848, self=0x7ffb47e039f0, tsdn=0x7ffb489fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffb489fecb8, shard=shard@entry=0x7ffb47e039d8, size=size@entry=462848, alignment=alignment@entry=64, slab=slab@entry=false, szind=55, zero=true, guarded=false, deferred_work_generated=0x7ffb489fd787) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffb489fecb8, arena=arena@entry=0x7ffb47e01040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffb489fecb8, arena=arena@entry=0x7ffb47e01040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffb489fecb8, arena=arena@entry=0x7ffb47e01040, usize=usize@entry=458752, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffb489fecb8, arena=0x7ffb47e01040, arena@entry=0x0, size=size@entry=422912, ind=ind@entry=55, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffb489ff010, zero=true, ind=55, size=422912, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffb489ff010, zero=true, ind=55, size=422912, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=55, usize=458752, size=422912, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff7389c13 in file_output(fieldstat_instance*, long long) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff7385b88 in fieldstat_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #24 0x00007ffeb65cae28 in rule_monitor_loop () from /opt/tsg/framework/lib/libmaat4.so.4 #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 123 (Thread 0x7ffb47bff700 (LWP 3947)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffced4f1228, mtx=mtx@entry=0x7ffced4f1200, tspec=tspec@entry=0x7ffb47bfe120) at tinycthread_extra.c:106 #3 0x00007ffeb7086452 in rd_kafka_q_serve (rkq=0x7ffced4f1200, timeout_ms=, max_cnt=max_cnt@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_CALLBACK, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:482 #4 0x00007ffeb7045eac in rd_kafka_thread_main (arg=arg@entry=0x7ffde8836000) at rdkafka.c:2058 #5 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #6 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 122 (Thread 0x7ffb473fe700 (LWP 3948)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffcec5620e8, mtx=mtx@entry=0x7ffcec5620c0, tspec=tspec@entry=0x7ffb473fcdc0) at tinycthread_extra.c:106 #3 0x00007ffeb7085e99 in rd_kafka_q_pop_serve (rkq=0x7ffcec5620c0, timeout_us=, version=version@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_RETURN, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:412 #4 0x00007ffeb7085f30 in rd_kafka_q_pop (rkq=, timeout_us=, version=version@entry=0) at rdkafka_queue.c:436 #5 0x00007ffeb706798f in rd_kafka_broker_ops_serve (rkb=rkb@entry=0x7ffe82303c00, timeout_us=) at rdkafka_broker.c:3377 #6 0x00007ffeb7067a83 in rd_kafka_broker_ops_io_serve (rkb=rkb@entry=0x7ffe82303c00, abs_timeout=, abs_timeout@entry=9747680555118) at rdkafka_broker.c:3433 #7 0x00007ffeb7069637 in rd_kafka_broker_internal_serve (abs_timeout=9747680555118, rkb=0x7ffe82303c00) at rdkafka_broker.c:3610 #8 rd_kafka_broker_serve (rkb=rkb@entry=0x7ffe82303c00, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5159 #9 0x00007ffeb7069c43 in rd_kafka_broker_thread_main (arg=arg@entry=0x7ffe82303c00) at rdkafka_broker.c:5297 #10 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #11 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 121 (Thread 0x7ffb46bfd700 (LWP 3949)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb45803a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb45803a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffb45803a28, tsdn=0x7ffb46bfccb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffb46bfccb8, pac=pac@entry=0x7ffb458039f0, ehooks=ehooks@entry=0x7ffb458000c0, ecache=ecache@entry=0x7ffb45803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=69632, alignment=64, zero=false, commit=0x7ffb46bfb35f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffb46bfccb8, pac=pac@entry=0x7ffb458039f0, ehooks=ehooks@entry=0x7ffb458000c0, ecache=ecache@entry=0x7ffb45803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=69632, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffb46bfccb8, pac=pac@entry=0x7ffb458039f0, ehooks=ehooks@entry=0x7ffb458000c0, size=size@entry=69632, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffb46bfccb8, self=0x7ffb458039f0, size=69632, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffb46bfb4e7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffb46bfb4e7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=69632, self=0x7ffb458039f0, tsdn=0x7ffb46bfccb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffb46bfccb8, shard=shard@entry=0x7ffb458039d8, size=size@entry=69632, alignment=alignment@entry=64, slab=slab@entry=false, szind=44, zero=false, guarded=false, deferred_work_generated=0x7ffb46bfb4e7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffb46bfccb8, arena=arena@entry=0x7ffb45801040, usize=usize@entry=65536, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffb46bfccb8, arena=arena@entry=0x7ffb45801040, usize=usize@entry=65536, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffb46bfccb8, arena=arena@entry=0x7ffb45801040, usize=usize@entry=65536, zero=zero@entry=false) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffb46bfccb8, arena=0x7ffb45801040, arena@entry=0x0, size=size@entry=65536, ind=ind@entry=44, zero=zero@entry=false) at include/jemalloc/internal/sz.h:251 #16 0x000000000047d7f2 in arena_malloc (slow_path=, tcache=0x7ffb46bfd010, zero=false, ind=44, size=65536, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffb46bfd010, zero=false, ind=44, size=65536, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=44, usize=65536, size=65536, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=65536) at src/jemalloc.c:2722 #22 0x000000000047e6ed in imalloc_fastpath (fallback_alloc=0x47cea0 , size=size@entry=65536) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:266 #23 malloc (size=size@entry=65536) at src/jemalloc.c:2746 #24 0x00007ffeb7155b12 in rd_malloc (sz=65536) at rd.h:127 #25 rd_kafka_snappy_init_env_sg (env=env@entry=0x7ffb46bfb750, sg=sg@entry=true) at snappy.c:1812 #26 0x00007ffeb70edc68 in rd_kafka_msgset_writer_compress_snappy (slice=slice@entry=0x7ffb46bfb9f0, ciov=ciov@entry=0x7ffb46bfb9e0, msetw=, msetw=) at rdkafka_msgset_writer.c:1086 #27 0x00007ffeb70ef44b in rd_kafka_msgset_writer_compress (outlenp=, msetw=0x7ffb46bfba90) at rdkafka_msgset_writer.c:1188 #28 rd_kafka_msgset_writer_finalize (MessageSetSizep=0x7ffb46bfbc88, msetw=0x7ffb46bfba90) at rdkafka_msgset_writer.c:1405 #29 rd_kafka_msgset_create_ProduceRequest (rkb=rkb@entry=0x7ffe82304800, rktp=rktp@entry=0x7ffb45e3d000, rkmq=rkmq@entry=0x7ffb45e3d0e8, pid=..., epoch_base_msgid=epoch_base_msgid@entry=0, MessageSetSizep=MessageSetSizep@entry=0x7ffb46bfbc88) at rdkafka_msgset_writer.c:1477 #30 0x00007ffeb70a3746 in rd_kafka_ProduceRequest (rkb=rkb@entry=0x7ffe82304800, rktp=rktp@entry=0x7ffb45e3d000, pid=..., epoch_base_msgid=epoch_base_msgid@entry=0) at rdkafka_request.c:3454 #31 0x00007ffeb705b7e4 in rd_kafka_toppar_producer_serve (rkb=rkb@entry=0x7ffe82304800, rktp=rktp@entry=0x7ffb45e3d000, pid=..., now=now@entry=9746784122900, next_wakeup=next_wakeup@entry=0x7ffb46bfbf00, do_timeout_scan=do_timeout_scan@entry=0 '\000', may_send=may_send@entry=1 '\001') at rdkafka_broker.c:3908 #32 0x00007ffeb7069276 in rd_kafka_broker_produce_toppars (do_timeout_scan=, next_wakeup=0x7ffb46bfbef8, now=, rkb=0x7ffe82304800) at rdkafka_broker.c:3971 #33 rd_kafka_broker_producer_serve (abs_timeout=9746784772173, rkb=0x7ffe82304800) at rdkafka_broker.c:4025 #34 rd_kafka_broker_serve (rkb=rkb@entry=0x7ffe82304800, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5164 #35 0x00007ffeb7069b65 in rd_kafka_broker_thread_main (arg=arg@entry=0x7ffe82304800) at rdkafka_broker.c:5328 #36 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #37 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #38 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 120 (Thread 0x7ffb455ff700 (LWP 3950)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7fface603a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7fface603a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7fface603a28, tsdn=0x7ffb455fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffb455fecb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, ecache=ecache@entry=0x7fface603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=233472, alignment=64, zero=true, commit=0x7ffb455fd84f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffb455fecb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, ecache=ecache@entry=0x7fface603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=233472, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffb455fecb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, size=size@entry=233472, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffb455fecb8, self=0x7fface6039f0, size=233472, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffb455fd9d7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffb455fd9d7, frequent_reuse=false, guarded=false, zero=true, alignment=, size=233472, self=0x7fface6039f0, tsdn=0x7ffb455fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffb455fecb8, shard=shard@entry=0x7fface6039d8, size=size@entry=233472, alignment=alignment@entry=64, slab=slab@entry=false, szind=51, zero=true, guarded=false, deferred_work_generated=0x7ffb455fd9d7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffb455fecb8, arena=arena@entry=0x7fface601040, usize=usize@entry=229376, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffb455fecb8, arena=arena@entry=0x7fface601040, usize=usize@entry=229376, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffb455fecb8, arena=arena@entry=0x7fface601040, usize=usize@entry=229376, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffb455fecb8, arena=0x7fface601040, arena@entry=0x0, size=size@entry=204800, ind=ind@entry=51, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffb455ff010, zero=true, ind=51, size=204800, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffb455ff010, zero=true, ind=51, size=204800, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=51, usize=229376, size=204800, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff7389c13 in file_output(fieldstat_instance*, long long) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff7385b88 in fieldstat_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #24 0x00007ffff7385bc8 in fieldstat_thread_schema_output(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 119 (Thread 0x7ffb44dfe700 (LWP 3951)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffdf5176209 in ?? () #3 0x0000000000000000 in ?? () Thread 118 (Thread 0x7ffb445fd700 (LWP 3952)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf6c14d68) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf6c14d68) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffdf6c14d68, tsdn=0x13d28) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=3, ptrs=0x7ffb445fba70, binind=, cache_bin=, tcache=, tsd=0x13d28) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4139863400, cache_bin=0x7ffb445fbab0, tcache=0x8, tsd=tsd@entry=0x13d28) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ffb445fccb8, tcache=tcache@entry=0x7ffb445fd010, cache_bin=cache_bin@entry=0x7ffb445fd108, binind=binind@entry=10, rem=) at src/tcache.c:529 #9 0x00000000005004e6 in tcache_gc_small (szind=10, tcache=0x7ffb445fd010, tcache_slow=0x7ffb445fcdb8, tsd=0x7ffb445fccb8) at src/tcache.c:148 #10 tcache_event (tsd=tsd@entry=0x7ffb445fccb8) at src/tcache.c:193 #11 je_tcache_gc_event_handler (tsd=tsd@entry=0x7ffb445fccb8, elapsed=elapsed@entry=18446744073709551615) at src/tcache.c:219 #12 0x0000000000502d9b in je_te_event_trigger (tsd=0x7ffb445fccb8, ctx=) at src/thread_event.c:299 #13 0x000000000047d4b1 in te_event_advance (is_alloc=true, usize=128, tsd=) at include/jemalloc/internal/thread_event.h:287 #14 thread_alloc_event (usize=128, tsd=) at include/jemalloc/internal/thread_event.h:298 #15 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2585 #16 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #17 je_malloc_default (size=size@entry=128) at src/jemalloc.c:2722 #18 0x0000000000503c29 in fallback_impl (size=128) at src/jemalloc_cpp.cpp:97 #19 0x00007ffff738ce2a in std::vector >::_M_insert_aux(__gnu_cxx::__normal_iterator > >, std::string const&) () from /opt/tsg/framework/lib/libfieldstat3.so #20 0x00007ffff738cc20 in line_protocol_dynamic_metric_output(fieldstat_dynamic_instance*) () from /opt/tsg/framework/lib/libfieldstat3.so #21 0x00007ffff7388280 in fieldstat_dynamic_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #22 0x00007ffff73882d8 in fieldstat_dynamic_thread_schema_output(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #24 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 117 (Thread 0x7ffb43dfc700 (LWP 3953)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf5615388) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf5615388) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffdf5615388, tsdn=0x14348) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=32, ptrs=0x7ffb43dfab70, binind=, cache_bin=, tcache=, tsd=0x14348) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4116796296, cache_bin=, tcache=, tsd=tsd@entry=0x14348) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ffb43dfbcb8, tcache=, cache_bin=, binind=binind@entry=17, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=17, ptr=0x7ff1096fd780, tcache=, tsd=0x7ffb43dfbcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ffb43dfc010, ptr=0x7ff1096fd780, tsd=0x7ffb43dfbcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff1096fd780) at src/jemalloc.c:3014 #14 0x00007ffff738cc80 in line_protocol_dynamic_metric_output(fieldstat_dynamic_instance*) () from /opt/tsg/framework/lib/libfieldstat3.so #15 0x00007ffff7388280 in fieldstat_dynamic_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #16 0x00007ffff73882d8 in fieldstat_dynamic_thread_schema_output(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #17 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #18 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 116 (Thread 0x7ffb433eb700 (LWP 3954)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7fface603a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7fface603a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7fface603a28, tsdn=0x7ffb433eacb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffb433eacb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, ecache=ecache@entry=0x7fface603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=266240, alignment=64, zero=true, commit=0x7ffb433e9c1f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffb433eacb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, ecache=ecache@entry=0x7fface603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=266240, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffb433eacb8, pac=pac@entry=0x7fface6039f0, ehooks=ehooks@entry=0x7fface6000c0, size=size@entry=266240, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffb433eacb8, self=0x7fface6039f0, size=266240, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffb433e9da7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffb433e9da7, frequent_reuse=false, guarded=false, zero=true, alignment=, size=266240, self=0x7fface6039f0, tsdn=0x7ffb433eacb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffb433eacb8, shard=shard@entry=0x7fface6039d8, size=size@entry=266240, alignment=alignment@entry=64, slab=slab@entry=false, szind=52, zero=true, guarded=false, deferred_work_generated=0x7ffb433e9da7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffb433eacb8, arena=arena@entry=0x7fface601040, usize=usize@entry=262144, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffb433eacb8, arena=arena@entry=0x7fface601040, usize=usize@entry=262144, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffb433eacb8, arena=arena@entry=0x7fface601040, usize=usize@entry=262144, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffb433eacb8, arena=0x7fface601040, arena@entry=0x0, size=size@entry=262144, ind=ind@entry=52, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffb433eb010, zero=true, ind=52, size=262144, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffb433eb010, zero=true, ind=52, size=262144, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=52, usize=262144, size=262144, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff75a522f in hdr_init () from /opt/tsg/framework/lib/libMESA_field_stat2.so #23 0x00007ffff759fb52 in FS_passive_output(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #24 0x00007ffff759fe08 in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 115 (Thread 0x7ffad66f3700 (LWP 3956)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa8803a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa8803a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffaa8803a28, tsdn=0x7ffad66f2cb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffad66f2cb8, pac=pac@entry=0x7ffaa88039f0, ehooks=ehooks@entry=0x7ffaa88000c0, ecache=ecache@entry=0x7ffaa8803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=266240, alignment=64, zero=true, commit=0x7ffad66f1c1f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffad66f2cb8, pac=pac@entry=0x7ffaa88039f0, ehooks=ehooks@entry=0x7ffaa88000c0, ecache=ecache@entry=0x7ffaa8803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=266240, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffad66f2cb8, pac=pac@entry=0x7ffaa88039f0, ehooks=ehooks@entry=0x7ffaa88000c0, size=size@entry=266240, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffad66f2cb8, self=0x7ffaa88039f0, size=266240, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffad66f1da7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffad66f1da7, frequent_reuse=false, guarded=false, zero=true, alignment=, size=266240, self=0x7ffaa88039f0, tsdn=0x7ffad66f2cb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffad66f2cb8, shard=shard@entry=0x7ffaa88039d8, size=size@entry=266240, alignment=alignment@entry=64, slab=slab@entry=false, szind=52, zero=true, guarded=false, deferred_work_generated=0x7ffad66f1da7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffad66f2cb8, arena=arena@entry=0x7ffaa8801040, usize=usize@entry=262144, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffad66f2cb8, arena=arena@entry=0x7ffaa8801040, usize=usize@entry=262144, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffad66f2cb8, arena=arena@entry=0x7ffaa8801040, usize=usize@entry=262144, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffad66f2cb8, arena=0x7ffaa8801040, arena@entry=0x0, size=size@entry=262144, ind=ind@entry=52, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffad66f3010, zero=true, ind=52, size=262144, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffad66f3010, zero=true, ind=52, size=262144, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=52, usize=262144, size=262144, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff75a522f in hdr_init () from /opt/tsg/framework/lib/libMESA_field_stat2.so #23 0x00007ffff759fb52 in FS_passive_output(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #24 0x00007ffff759fe08 in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 114 (Thread 0x7ffad5ef2700 (LWP 3957)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa8414908) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa8414908) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ffad5ef1cb8, arena=arena@entry=0x7ffaa8401040, cache_bin=cache_bin@entry=0x7ffad5ef2090, cache_bin_info=, binind=binind@entry=5, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ffad5ef1cb8, arena=arena@entry=0x7ffaa8401040, tcache=tcache@entry=0x7ffad5ef2010, cache_bin=cache_bin@entry=0x7ffad5ef2090, binind=binind@entry=5, tcache_success=tcache_success@entry=0x7ffad5ef1160) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=5, size=, tcache=0x7ffad5ef2010, arena=0x7ffaa8401040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ffad5ef2010, zero=false, ind=5, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffad5ef2010, zero=false, ind=5, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=5, usize=80, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x00007ffff595a3b1 in calloc () from /lib64/libc.so.6 #16 0x00007ffad66f5932 in MVSketch::MVSketch(int, int, int) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #17 0x00007ffad66f554c in MV_Sketch_query(void*, double, int (*)(unsigned char const*, int, long, long, long, void*), void*) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #18 0x00007ffad68fbdec in ?? () #19 0x00007ffa9d0b04c0 in ?? () #20 0x0000001e00000000 in ?? () #21 0x0000000000000000 in ?? () Thread 113 (Thread 0x7fface3ff700 (LWP 3958)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa2403a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa2403a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffaa2403a28, tsdn=0x7fface3fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7fface3fecb8, pac=pac@entry=0x7ffaa24039f0, ehooks=ehooks@entry=0x7ffaa24000c0, ecache=ecache@entry=0x7ffaa2403a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, commit=0x7fface3fde6f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7fface3fecb8, pac=pac@entry=0x7ffaa24039f0, ehooks=ehooks@entry=0x7ffaa24000c0, ecache=ecache@entry=0x7ffaa2403a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7fface3fecb8, pac=pac@entry=0x7ffaa24039f0, ehooks=ehooks@entry=0x7ffaa24000c0, size=size@entry=462848, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7fface3fecb8, self=0x7ffaa24039f0, size=462848, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7fface3fdff7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7fface3fdff7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=462848, self=0x7ffaa24039f0, tsdn=0x7fface3fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7fface3fecb8, shard=shard@entry=0x7ffaa24039d8, size=size@entry=462848, alignment=alignment@entry=64, slab=slab@entry=false, szind=55, zero=false, guarded=false, deferred_work_generated=0x7fface3fdff7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7fface3fecb8, arena=arena@entry=0x7ffaa2401040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7fface3fecb8, arena=arena@entry=0x7ffaa2401040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7fface3fecb8, arena=arena@entry=0x7ffaa2401040, usize=usize@entry=458752, zero=zero@entry=false) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7fface3fecb8, arena=0x7ffaa2401040, arena@entry=0x0, size=size@entry=437120, ind=ind@entry=55, zero=zero@entry=false) at include/jemalloc/internal/sz.h:251 #16 0x000000000047d7f2 in arena_malloc (slow_path=, tcache=0x7fface3ff010, zero=false, ind=55, size=437120, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7fface3ff010, zero=false, ind=55, size=437120, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=55, usize=458752, size=437120, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=size@entry=437120) at src/jemalloc.c:2722 #22 0x0000000000503c29 in fallback_impl (size=437120) at src/jemalloc_cpp.cpp:97 #23 0x00007ffff6219289 in operator new[](unsigned long) () from /lib64/libstdc++.so.6 #24 0x00007ffad66f5891 in MVSketch::MVSketch(int, int, int) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #25 0x00007ffad66f554c in MV_Sketch_query(void*, double, int (*)(unsigned char const*, int, long, long, long, void*), void*) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #26 0x00007ffad68fbdec in ?? () #27 0x00007ffaa2266880 in ?? () #28 0x0000000000000000 in ?? () Thread 112 (Thread 0x7ffacdbfe700 (LWP 3959)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa9b603a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa9b603a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffa9b603a28, tsdn=0x7ffacdbfdcb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffacdbfdcb8, pac=pac@entry=0x7ffa9b6039f0, ehooks=ehooks@entry=0x7ffa9b6000c0, ecache=ecache@entry=0x7ffa9b603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, commit=0x7ffacdbfce6f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffacdbfdcb8, pac=pac@entry=0x7ffa9b6039f0, ehooks=ehooks@entry=0x7ffa9b6000c0, ecache=ecache@entry=0x7ffa9b603a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffacdbfdcb8, pac=pac@entry=0x7ffa9b6039f0, ehooks=ehooks@entry=0x7ffa9b6000c0, size=size@entry=462848, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffacdbfdcb8, self=0x7ffa9b6039f0, size=462848, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffacdbfcff7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffacdbfcff7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=462848, self=0x7ffa9b6039f0, tsdn=0x7ffacdbfdcb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffacdbfdcb8, shard=shard@entry=0x7ffa9b6039d8, size=size@entry=462848, alignment=alignment@entry=64, slab=slab@entry=false, szind=55, zero=false, guarded=false, deferred_work_generated=0x7ffacdbfcff7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffacdbfdcb8, arena=arena@entry=0x7ffa9b601040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffacdbfdcb8, arena=arena@entry=0x7ffa9b601040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffacdbfdcb8, arena=arena@entry=0x7ffa9b601040, usize=usize@entry=458752, zero=zero@entry=false) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffacdbfdcb8, arena=0x7ffa9b601040, arena@entry=0x0, size=size@entry=437120, ind=ind@entry=55, zero=zero@entry=false) at include/jemalloc/internal/sz.h:251 #16 0x000000000047d7f2 in arena_malloc (slow_path=, tcache=0x7ffacdbfe010, zero=false, ind=55, size=437120, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffacdbfe010, zero=false, ind=55, size=437120, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=55, usize=458752, size=437120, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=size@entry=437120) at src/jemalloc.c:2722 #22 0x0000000000503c29 in fallback_impl (size=437120) at src/jemalloc_cpp.cpp:97 #23 0x00007ffff6219289 in operator new[](unsigned long) () from /lib64/libstdc++.so.6 #24 0x00007ffad66f5891 in MVSketch::MVSketch(int, int, int) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #25 0x00007ffad66f554c in MV_Sketch_query(void*, double, int (*)(unsigned char const*, int, long, long, long, void*), void*) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #26 0x00007ffad68fbdec in ?? () #27 0x00007ffa9b026800 in ?? () #28 0x0000000000000000 in ?? () Thread 111 (Thread 0x7ffacd3fd700 (LWP 3960)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa95003a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa95003a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffa95003a28, tsdn=0x7ffacd3fccb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffacd3fccb8, pac=pac@entry=0x7ffa950039f0, ehooks=ehooks@entry=0x7ffa950000c0, ecache=ecache@entry=0x7ffa95003a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, commit=0x7ffacd3fbe6f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffacd3fccb8, pac=pac@entry=0x7ffa950039f0, ehooks=ehooks@entry=0x7ffa950000c0, ecache=ecache@entry=0x7ffa95003a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffacd3fccb8, pac=pac@entry=0x7ffa950039f0, ehooks=ehooks@entry=0x7ffa950000c0, size=size@entry=462848, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffacd3fccb8, self=0x7ffa950039f0, size=462848, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffacd3fbff7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffacd3fbff7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=462848, self=0x7ffa950039f0, tsdn=0x7ffacd3fccb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffacd3fccb8, shard=shard@entry=0x7ffa950039d8, size=size@entry=462848, alignment=alignment@entry=64, slab=slab@entry=false, szind=55, zero=false, guarded=false, deferred_work_generated=0x7ffacd3fbff7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffacd3fccb8, arena=arena@entry=0x7ffa95001040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffacd3fccb8, arena=arena@entry=0x7ffa95001040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffacd3fccb8, arena=arena@entry=0x7ffa95001040, usize=usize@entry=458752, zero=zero@entry=false) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffacd3fccb8, arena=0x7ffa95001040, arena@entry=0x0, size=size@entry=437120, ind=ind@entry=55, zero=zero@entry=false) at include/jemalloc/internal/sz.h:251 #16 0x000000000047d7f2 in arena_malloc (slow_path=, tcache=0x7ffacd3fd010, zero=false, ind=55, size=437120, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffacd3fd010, zero=false, ind=55, size=437120, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=55, usize=458752, size=437120, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=size@entry=437120) at src/jemalloc.c:2722 #22 0x0000000000503c29 in fallback_impl (size=437120) at src/jemalloc_cpp.cpp:97 #23 0x00007ffff6219289 in operator new[](unsigned long) () from /lib64/libstdc++.so.6 #24 0x00007ffad66f5891 in MVSketch::MVSketch(int, int, int) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #25 0x00007ffad66f554c in MV_Sketch_query(void*, double, int (*)(unsigned char const*, int, long, long, long, void*), void*) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #26 0x00007ffad68fbdec in ?? () #27 0x00007ffa94cb56c0 in ?? () #28 0x0000000000000000 in ?? () Thread 110 (Thread 0x7ffaccbfc700 (LWP 3961)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa8e203a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa8e203a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffa8e203a28, tsdn=0x7ffaccbfbcb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffaccbfbcb8, pac=pac@entry=0x7ffa8e2039f0, ehooks=ehooks@entry=0x7ffa8e2000c0, ecache=ecache@entry=0x7ffa8e203a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, commit=0x7ffaccbfae6f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffaccbfbcb8, pac=pac@entry=0x7ffa8e2039f0, ehooks=ehooks@entry=0x7ffa8e2000c0, ecache=ecache@entry=0x7ffa8e203a28, expand_edata=expand_edata@entry=0x0, size=size@entry=462848, alignment=64, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffaccbfbcb8, pac=pac@entry=0x7ffa8e2039f0, ehooks=ehooks@entry=0x7ffa8e2000c0, size=size@entry=462848, alignment=alignment@entry=64, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffaccbfbcb8, self=0x7ffa8e2039f0, size=462848, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffaccbfaff7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffaccbfaff7, frequent_reuse=false, guarded=false, zero=false, alignment=, size=462848, self=0x7ffa8e2039f0, tsdn=0x7ffaccbfbcb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffaccbfbcb8, shard=shard@entry=0x7ffa8e2039d8, size=size@entry=462848, alignment=alignment@entry=64, slab=slab@entry=false, szind=55, zero=false, guarded=false, deferred_work_generated=0x7ffaccbfaff7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffaccbfbcb8, arena=arena@entry=0x7ffa8e201040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffaccbfbcb8, arena=arena@entry=0x7ffa8e201040, usize=usize@entry=458752, alignment=alignment@entry=64, zero=zero@entry=false) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffaccbfbcb8, arena=arena@entry=0x7ffa8e201040, usize=usize@entry=458752, zero=zero@entry=false) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffaccbfbcb8, arena=0x7ffa8e201040, arena@entry=0x0, size=size@entry=437120, ind=ind@entry=55, zero=zero@entry=false) at include/jemalloc/internal/sz.h:251 #16 0x000000000047d7f2 in arena_malloc (slow_path=, tcache=0x7ffaccbfc010, zero=false, ind=55, size=437120, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffaccbfc010, zero=false, ind=55, size=437120, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=55, usize=458752, size=437120, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=size@entry=437120) at src/jemalloc.c:2722 #22 0x0000000000503c29 in fallback_impl (size=437120) at src/jemalloc_cpp.cpp:97 #23 0x00007ffff6219289 in operator new[](unsigned long) () from /lib64/libstdc++.so.6 #24 0x00007ffad66f5891 in MVSketch::MVSketch(int, int, int) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #25 0x00007ffad66f554c in MV_Sketch_query(void*, double, int (*)(unsigned char const*, int, long, long, long, void*), void*) () from /opt/tsg/framework/lib/libMV_sketch.so.2 #26 0x00007ffad68fbdec in ?? () #27 0x00007ffa8e0f8a80 in ?? () #28 0x0000000000000000 in ?? () Thread 109 (Thread 0x7ffacb25f700 (LWP 3962)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffff759fe0f in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 108 (Thread 0x7ffac9d98700 (LWP 3963)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffff759fe0f in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 107 (Thread 0x7ffac9393700 (LWP 3964)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffff759fe0f in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 106 (Thread 0x7ffac7256700 (LWP 3965)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x0) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 105 (Thread 0x7ffac6a55700 (LWP 3966)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 104 (Thread 0x7ffac6254700 (LWP 3967)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x2) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 103 (Thread 0x7ffac5a53700 (LWP 3968)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x3) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 102 (Thread 0x7ffac5252700 (LWP 3969)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x4) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 101 (Thread 0x7ffac4a51700 (LWP 3970)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x5) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 100 (Thread 0x7ffac4250700 (LWP 3971)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x6) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 99 (Thread 0x7ffac3a4f700 (LWP 3972)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x7) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 98 (Thread 0x7ffac324e700 (LWP 3973)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x8) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 97 (Thread 0x7ffac2a4d700 (LWP 3974)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x9) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 96 (Thread 0x7ffac224c700 (LWP 3975)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xa) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 95 (Thread 0x7ffac1a4b700 (LWP 3976)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xb) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 94 (Thread 0x7ffac124a700 (LWP 3977)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xc) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 93 (Thread 0x7ffac0a49700 (LWP 3978)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xd) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 92 (Thread 0x7ffac0248700 (LWP 3979)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xe) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 91 (Thread 0x7ffabfa47700 (LWP 3980)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0xf) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 90 (Thread 0x7ffabf246700 (LWP 3981)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x10) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 89 (Thread 0x7ffabea45700 (LWP 3982)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x11) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 88 (Thread 0x7ffabe244700 (LWP 3983)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x12) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 87 (Thread 0x7ffabda43700 (LWP 3984)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x13) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 86 (Thread 0x7ffabd242700 (LWP 3985)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x14) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 85 (Thread 0x7ffabca41700 (LWP 3986)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x15) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 84 (Thread 0x7ffabc240700 (LWP 3987)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x16) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 83 (Thread 0x7ffabba3f700 (LWP 3988)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x17) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 82 (Thread 0x7ffabb23e700 (LWP 3989)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x18) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 81 (Thread 0x7ffabaa3d700 (LWP 3990)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x19) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 80 (Thread 0x7ffaba23c700 (LWP 3991)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1a) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 79 (Thread 0x7ffab9a3b700 (LWP 3992)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1b) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 78 (Thread 0x7ffab923a700 (LWP 3993)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1c) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 77 (Thread 0x7ffab8a39700 (LWP 3994)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1d) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 76 (Thread 0x7ffab8238700 (LWP 3995)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1e) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 75 (Thread 0x7ffab7a37700 (LWP 3996)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x1f) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 74 (Thread 0x7ffab7236700 (LWP 3997)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x20) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 73 (Thread 0x7ffab6a35700 (LWP 3998)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x21) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 72 (Thread 0x7ffab6234700 (LWP 3999)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x22) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 71 (Thread 0x7ffab5a33700 (LWP 4000)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x23) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 70 (Thread 0x7ffab5232700 (LWP 4001)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x24) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 69 (Thread 0x7ffab4a31700 (LWP 4002)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x25) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 68 (Thread 0x7ffab4230700 (LWP 4003)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x26) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 67 (Thread 0x7ffab3a2f700 (LWP 4004)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x27) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 66 (Thread 0x7ffab322e700 (LWP 4005)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x28) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 65 (Thread 0x7ffab2a2d700 (LWP 4006)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x29) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 64 (Thread 0x7ffab222c700 (LWP 4007)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffac725edc3 in thread_work (ptr=0x2a) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/LockFreeThreadPool.cpp:166 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 63 (Thread 0x7ffab1a2b700 (LWP 4008)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa9803a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa9803a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffaa9803a28, tsdn=0x7ffab1a2acb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffab1a2acb8, pac=pac@entry=0x7ffaa98039f0, ehooks=ehooks@entry=0x7ffaa98000c0, ecache=ecache@entry=0x7ffaa9803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=45056, alignment=64, zero=true, commit=0x7ffab1a29c1f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffab1a2acb8, pac=pac@entry=0x7ffaa98039f0, ehooks=ehooks@entry=0x7ffaa98000c0, ecache=ecache@entry=0x7ffaa9803a28, expand_edata=expand_edata@entry=0x0, size=size@entry=45056, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffab1a2acb8, pac=pac@entry=0x7ffaa98039f0, ehooks=ehooks@entry=0x7ffaa98000c0, size=size@entry=45056, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffab1a2acb8, self=0x7ffaa98039f0, size=45056, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffab1a29da7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffab1a29da7, frequent_reuse=false, guarded=false, zero=true, alignment=, size=45056, self=0x7ffaa98039f0, tsdn=0x7ffab1a2acb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffab1a2acb8, shard=shard@entry=0x7ffaa98039d8, size=size@entry=45056, alignment=alignment@entry=64, slab=slab@entry=false, szind=41, zero=true, guarded=false, deferred_work_generated=0x7ffab1a29da7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffab1a2acb8, arena=arena@entry=0x7ffaa9801040, usize=usize@entry=40960, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffab1a2acb8, arena=arena@entry=0x7ffaa9801040, usize=usize@entry=40960, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffab1a2acb8, arena=arena@entry=0x7ffaa9801040, usize=usize@entry=40960, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffab1a2acb8, arena=0x7ffaa9801040, arena@entry=0x0, size=size@entry=40960, ind=ind@entry=41, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffab1a2b010, zero=true, ind=41, size=40960, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffab1a2b010, zero=true, ind=41, size=40960, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=41, usize=40960, size=40960, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff75a522f in hdr_init () from /opt/tsg/framework/lib/libMESA_field_stat2.so #23 0x00007ffff759fb52 in FS_passive_output(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #24 0x00007ffff759fe08 in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 62 (Thread 0x7ffab122a700 (LWP 4009)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffac725f677 in fs2_statistics (ptr=) at /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/pxz/hos/src/hos.cpp:1050 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 61 (Thread 0x7ffab0a29700 (LWP 4010)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff5999894 in sleep () from /lib64/libc.so.6 #2 0x00007ffff759fe0f in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 60 (Thread 0x7ffaaf628700 (LWP 4011)): #0 0x00007ffff59d30e3 in epoll_wait () from /lib64/libc.so.6 #1 0x00007ffebf5fb84e in __epoll_event_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #2 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 59 (Thread 0x7ffaaee27700 (LWP 4012)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaae414828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaae414828) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ffaaee26cb8, arena=arena@entry=0x7ffaae401040, cache_bin=cache_bin@entry=0x7ffaaee27078, cache_bin_info=, binind=binind@entry=4, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ffaaee26cb8, arena=arena@entry=0x7ffaae401040, tcache=tcache@entry=0x7ffaaee27010, cache_bin=cache_bin@entry=0x7ffaaee27078, binind=binind@entry=4, tcache_success=tcache_success@entry=0x7ffaaee1d9c0) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=4, size=, tcache=0x7ffaaee27010, arena=0x7ffaae401040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ffaaee27010, zero=false, ind=4, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffaaee27010, zero=false, ind=4, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=4, usize=64, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x00007ffebf5f818f in cJSON_New_Item () from /opt/tsg/mrzcpd/lib/libmarsio.so #16 0x00007ffebf5fa5bf in cJSON_CreateNumber () from /opt/tsg/mrzcpd/lib/libmarsio.so #17 0x00007ffebebeb7a2 in __create_uint64_array () from /opt/tsg/mrzcpd/lib/libmarsio.so #18 0x00007ffebebebe94 in __create_vdev_stats () from /opt/tsg/mrzcpd/lib/libmarsio.so #19 0x00007ffebebec791 in mrapp_monit_loop () from /opt/tsg/mrzcpd/lib/libmarsio.so #20 0x00007ffebebda1c0 in mrapp_ctrlplane_thread () from /opt/tsg/mrzcpd/lib/libmarsio.so #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 58 (Thread 0x7ffaae1ff700 (LWP 4013)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa9403a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa9403a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffaa9403a28, tsdn=0x7ffaae1fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ffaae1fecb8, pac=pac@entry=0x7ffaa94039f0, ehooks=ehooks@entry=0x7ffaa94000c0, ecache=ecache@entry=0x7ffaa9403a28, expand_edata=expand_edata@entry=0x0, size=size@entry=118784, alignment=64, zero=true, commit=0x7ffaae1fdc9f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ffaae1fecb8, pac=pac@entry=0x7ffaa94039f0, ehooks=ehooks@entry=0x7ffaa94000c0, ecache=ecache@entry=0x7ffaa9403a28, expand_edata=expand_edata@entry=0x0, size=size@entry=118784, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ffaae1fecb8, pac=pac@entry=0x7ffaa94039f0, ehooks=ehooks@entry=0x7ffaa94000c0, size=size@entry=118784, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ffaae1fecb8, self=0x7ffaa94039f0, size=118784, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ffaae1fde27) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ffaae1fde27, frequent_reuse=false, guarded=false, zero=true, alignment=, size=118784, self=0x7ffaa94039f0, tsdn=0x7ffaae1fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ffaae1fecb8, shard=shard@entry=0x7ffaa94039d8, size=size@entry=118784, alignment=alignment@entry=64, slab=slab@entry=false, szind=47, zero=true, guarded=false, deferred_work_generated=0x7ffaae1fde27) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ffaae1fecb8, arena=arena@entry=0x7ffaa9401040, usize=usize@entry=114688, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ffaae1fecb8, arena=arena@entry=0x7ffaa9401040, usize=usize@entry=114688, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ffaae1fecb8, arena=arena@entry=0x7ffaa9401040, usize=usize@entry=114688, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ffaae1fecb8, arena=0x7ffaa9401040, arena@entry=0x0, size=size@entry=105472, ind=ind@entry=47, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ffaae1ff010, zero=true, ind=47, size=105472, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ffaae1ff010, zero=true, ind=47, size=105472, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=47, usize=114688, size=105472, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff759f2ec in FS_passive_output(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #23 0x00007ffff759fe08 in fs2_thread_screen_print(void*) () from /opt/tsg/framework/lib/libMESA_field_stat2.so #24 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #25 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 57 (Thread 0x7ffaad9fe700 (LWP 4014)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x00007ffff73882e9 in fieldstat_dynamic_thread_schema_output(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 56 (Thread 0x7ffaad1fd700 (LWP 4015)): #0 0x00007ffff59d37bd in accept () from /lib64/libc.so.6 #1 0x00007ffaafc076f0 in ?? () #2 0x0000000000000000 in ?? () Thread 55 (Thread 0x7ffaac9fc700 (LWP 4016)): #0 0x00007ffff59d3a23 in recvfrom () from /lib64/libc.so.6 #1 0x00007ffaafc00d2a in ?? () #2 0x00000000000009ab in ?? () #3 0x0000000000000000 in ?? () Thread 54 (Thread 0x7ffaaa9ff700 (LWP 4018)): #0 0x00007ffff59d39bb in recv () from /lib64/libc.so.6 #1 0x00007ffaafc07b41 in ?? () #2 0x0000000000000000 in ?? () Thread 53 (Thread 0x7ff968eff700 (LWP 4019)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffeb3c03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffeb3c03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffeb3c03a28, tsdn=0x7ff968efecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ff968efecb8, pac=pac@entry=0x7ffeb3c039f0, ehooks=ehooks@entry=0x7ffeb3c000c0, ecache=ecache@entry=0x7ffeb3c03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=61440, alignment=64, zero=true, commit=0x7ff968efd5ff, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ff968efecb8, pac=pac@entry=0x7ffeb3c039f0, ehooks=ehooks@entry=0x7ffeb3c000c0, ecache=ecache@entry=0x7ffeb3c03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=61440, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ff968efecb8, pac=pac@entry=0x7ffeb3c039f0, ehooks=ehooks@entry=0x7ffeb3c000c0, size=size@entry=61440, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ff968efecb8, self=0x7ffeb3c039f0, size=61440, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ff968efd787) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ff968efd787, frequent_reuse=false, guarded=false, zero=true, alignment=, size=61440, self=0x7ffeb3c039f0, tsdn=0x7ff968efecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ff968efecb8, shard=shard@entry=0x7ffeb3c039d8, size=size@entry=61440, alignment=alignment@entry=64, slab=slab@entry=false, szind=43, zero=true, guarded=false, deferred_work_generated=0x7ff968efd787) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ff968efecb8, arena=arena@entry=0x7ffeb3c01040, usize=usize@entry=57344, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ff968efecb8, arena=arena@entry=0x7ffeb3c01040, usize=usize@entry=57344, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ff968efecb8, arena=arena@entry=0x7ffeb3c01040, usize=usize@entry=57344, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ff968efecb8, arena=0x7ffeb3c01040, arena@entry=0x0, size=size@entry=54272, ind=ind@entry=43, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ff968eff010, zero=true, ind=43, size=54272, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ff968eff010, zero=true, ind=43, size=54272, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=43, usize=57344, size=54272, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff7389c13 in file_output(fieldstat_instance*, long long) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff7385b88 in fieldstat_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #24 0x00007ffeb65cae28 in rule_monitor_loop () from /opt/tsg/framework/lib/libmaat4.so.4 #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 52 (Thread 0x7ff967cf7700 (LWP 4021)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffdec418528, mtx=mtx@entry=0x7ffdec418500, tspec=tspec@entry=0x7ff967cf6120) at tinycthread_extra.c:106 #3 0x00007ffeb7086452 in rd_kafka_q_serve (rkq=0x7ffdec418500, timeout_ms=, max_cnt=max_cnt@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_CALLBACK, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:482 #4 0x00007ffeb7045eac in rd_kafka_thread_main (arg=arg@entry=0x7ffe837f7000) at rdkafka.c:2058 #5 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #6 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 51 (Thread 0x7ff9674f6700 (LWP 4023)): #0 0x00007ffff6912de2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007ffeb70dea69 in cnd_timedwait (cond=, mtx=, ts=) at tinycthread.c:462 #2 0x00007ffeb70dee3d in cnd_timedwait_abs (cnd=cnd@entry=0x7ffdec418468, mtx=mtx@entry=0x7ffdec418440, tspec=tspec@entry=0x7ff9674f4dc0) at tinycthread_extra.c:106 #3 0x00007ffeb7085e99 in rd_kafka_q_pop_serve (rkq=0x7ffdec418440, timeout_us=, version=version@entry=0, cb_type=cb_type@entry=RD_KAFKA_Q_CB_RETURN, callback=callback@entry=0x0, opaque=opaque@entry=0x0) at rdkafka_queue.c:412 #4 0x00007ffeb7085f30 in rd_kafka_q_pop (rkq=, timeout_us=, version=version@entry=0) at rdkafka_queue.c:436 #5 0x00007ffeb706798f in rd_kafka_broker_ops_serve (rkb=rkb@entry=0x7ffde79da000, timeout_us=) at rdkafka_broker.c:3377 #6 0x00007ffeb7067a83 in rd_kafka_broker_ops_io_serve (rkb=rkb@entry=0x7ffde79da000, abs_timeout=, abs_timeout@entry=9747680554420) at rdkafka_broker.c:3433 #7 0x00007ffeb7069637 in rd_kafka_broker_internal_serve (abs_timeout=9747680554420, rkb=0x7ffde79da000) at rdkafka_broker.c:3610 #8 rd_kafka_broker_serve (rkb=rkb@entry=0x7ffde79da000, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5159 #9 0x00007ffeb7069c43 in rd_kafka_broker_thread_main (arg=arg@entry=0x7ffde79da000) at rdkafka_broker.c:5297 #10 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #11 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 50 (Thread 0x7ff9669ff700 (LWP 4024)): #0 0x00007ffff59c7ddd in poll () from /lib64/libc.so.6 #1 0x00007ffeb707f38e in rd_kafka_transport_poll (rktrans=rktrans@entry=0x7ff96560b000, tmout=tmout@entry=1000) at rdkafka_transport.c:977 #2 0x00007ffeb707f424 in rd_kafka_transport_io_serve (rktrans=0x7ff96560b000, timeout_ms=1000) at rdkafka_transport.c:809 #3 0x00007ffeb7067a71 in rd_kafka_broker_ops_io_serve (rkb=rkb@entry=0x7ffde79dac00, abs_timeout=) at rdkafka_broker.c:3425 #4 0x00007ffeb7069320 in rd_kafka_broker_producer_serve (abs_timeout=9747680594860, rkb=0x7ffde79dac00) at rdkafka_broker.c:4032 #5 rd_kafka_broker_serve (rkb=rkb@entry=0x7ffde79dac00, timeout_ms=, timeout_ms@entry=1000) at rdkafka_broker.c:5164 #6 0x00007ffeb7069b65 in rd_kafka_broker_thread_main (arg=arg@entry=0x7ffde79dac00) at rdkafka_broker.c:5328 #7 0x00007ffeb70de8d7 in _thrd_wrapper_function (aArg=) at tinycthread.c:576 #8 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #9 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 49 (Thread 0x7ff9655ff700 (LWP 4025)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf7a03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf7a03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ffdf7a03a28, tsdn=0x7ff9655fecb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ff9655fecb8, pac=pac@entry=0x7ffdf7a039f0, ehooks=ehooks@entry=0x7ffdf7a000c0, ecache=ecache@entry=0x7ffdf7a03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=86016, alignment=64, zero=true, commit=0x7ff9655fd84f, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ff9655fecb8, pac=pac@entry=0x7ffdf7a039f0, ehooks=ehooks@entry=0x7ffdf7a000c0, ecache=ecache@entry=0x7ffdf7a03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=86016, alignment=64, zero=true, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ff9655fecb8, pac=pac@entry=0x7ffdf7a039f0, ehooks=ehooks@entry=0x7ffdf7a000c0, size=size@entry=86016, alignment=alignment@entry=64, zero=zero@entry=true, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ff9655fecb8, self=0x7ffdf7a039f0, size=86016, alignment=64, zero=, guarded=, frequent_reuse=false, deferred_work_generated=0x7ff9655fd9d7) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ff9655fd9d7, frequent_reuse=false, guarded=false, zero=true, alignment=, size=86016, self=0x7ffdf7a039f0, tsdn=0x7ff9655fecb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ff9655fecb8, shard=shard@entry=0x7ffdf7a039d8, size=size@entry=86016, alignment=alignment@entry=64, slab=slab@entry=false, szind=45, zero=true, guarded=false, deferred_work_generated=0x7ff9655fd9d7) at src/pa.c:139 #12 0x000000000048bd54 in je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7ff9655fecb8, arena=arena@entry=0x7ffdf7a01040, usize=usize@entry=81920, alignment=alignment@entry=64, zero=) at src/arena.c:338 #13 0x00000000004e548e in je_large_palloc (tsdn=tsdn@entry=0x7ff9655fecb8, arena=arena@entry=0x7ffdf7a01040, usize=usize@entry=81920, alignment=alignment@entry=64, zero=zero@entry=true) at include/jemalloc/internal/arena_inlines_b.h:22 #14 0x00000000004e594e in je_large_malloc (tsdn=tsdn@entry=0x7ff9655fecb8, arena=arena@entry=0x7ffdf7a01040, usize=usize@entry=81920, zero=zero@entry=true) at src/large.c:17 #15 0x000000000048fbcd in je_arena_malloc_hard (tsdn=tsdn@entry=0x7ff9655fecb8, arena=0x7ffdf7a01040, arena@entry=0x0, size=size@entry=69632, ind=ind@entry=45, zero=zero@entry=true) at include/jemalloc/internal/sz.h:251 #16 0x000000000047ff9b in arena_malloc (slow_path=, tcache=0x7ff9655ff010, zero=true, ind=45, size=69632, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:162 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ff9655ff010, zero=true, ind=45, size=69632, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=45, usize=81920, size=69632, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 calloc (num=, size=) at src/jemalloc.c:2852 #22 0x00007ffff7389c13 in file_output(fieldstat_instance*, long long) () from /opt/tsg/framework/lib/libfieldstat3.so #23 0x00007ffff7385b88 in fieldstat_passive_output () from /opt/tsg/framework/lib/libfieldstat3.so #24 0x00007ffff7385bc8 in fieldstat_thread_schema_output(void*) () from /opt/tsg/framework/lib/libfieldstat3.so #25 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 48 (Thread 0x7ff964dfe700 (LWP 4031)): #0 0x00007ffff6915e9d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000000441648 in sapp_usleep (usec=usec@entry=1) at /home/yangwei/platform/sapp/src/dealpkt/plug_support.c:2144 #2 0x000000000046009c in sapp_time_loop_thread (arg=) at /home/yangwei/platform/sapp/src/timer/sapp_timer.c:509 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 47 (Thread 0x7ff9645fd700 (LWP 4032)): #0 0x00007ffff6915e9d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000000441648 in sapp_usleep (usec=usec@entry=10) at /home/yangwei/platform/sapp/src/dealpkt/plug_support.c:2144 #2 0x00000000004601a7 in sapp_time_event_thread () at /home/yangwei/platform/sapp/src/timer/sapp_timer.c:557 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 46 (Thread 0x7ff963dfc700 (LWP 4033)): #0 0x00007ffff6915e9d in nanosleep () from /lib64/libpthread.so.0 #1 0x0000000000441648 in sapp_usleep (usec=usec@entry=10) at /home/yangwei/platform/sapp/src/dealpkt/plug_support.c:2144 #2 0x000000000046f27a in cl_bg_thread (_arg_h=0x7fff1ae131c0) at /home/yangwei/platform/sapp/src/support/cpu_limit/cpu_limit.c:461 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 45 (Thread 0x7ff9635fb700 (LWP 4035)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff961a14748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff961a14748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ff961a14748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=46, ptrs=0x7ff9635f6ff0, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1637959496, cache_bin=0x7ff9635f7030, tcache=0x20, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9635facb8, tcache=tcache@entry=0x7ff9635fb010, cache_bin=cache_bin@entry=0x7ff9635fb060, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x00000000005004e6 in tcache_gc_small (szind=3, tcache=0x7ff9635fb010, tcache_slow=0x7ff9635fadb8, tsd=0x7ff9635facb8) at src/tcache.c:148 #10 tcache_event (tsd=tsd@entry=0x7ff9635facb8) at src/tcache.c:193 #11 je_tcache_gc_event_handler (tsd=tsd@entry=0x7ff9635facb8, elapsed=elapsed@entry=18446744073709551615) at src/tcache.c:219 #12 0x0000000000502d9b in je_te_event_trigger (tsd=0x7ff9635facb8, ctx=) at src/thread_event.c:299 #13 0x000000000047d4b1 in te_event_advance (is_alloc=true, usize=2048, tsd=) at include/jemalloc/internal/thread_event.h:287 #14 thread_alloc_event (usize=2048, tsd=) at include/jemalloc/internal/thread_event.h:298 #15 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2585 #16 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #17 je_malloc_default (size=) at src/jemalloc.c:2722 #18 0x00007ffff59e8f7e in backtrace_symbols () from /lib64/libc.so.6 #19 0x00000000004690c3 in signal_usr2_handler (signo=) at /home/yangwei/platform/sapp/src/sapp_dev/sapp_plug.c:85 #20 #21 0x00000000004d502e in edata_cmp_summary_comp (b=..., a=...) at include/jemalloc/internal/ph.h:293 #22 edata_snad_comp (b=, a=) at include/jemalloc/internal/edata.h:679 #23 edata_heap_ph_cmp (b=, a=) at src/edata.c:6 #24 phn_merge (cmp=, offset=, phn1=, phn0=) at include/jemalloc/internal/ph.h:142 #25 ph_try_aux_merge_pair (cmp=, offset=40, ph=0x7ff961a14810) at include/jemalloc/internal/ph.h:298 #26 ph_insert (cmp=, offset=40, phn=, ph=0x7ff961a14810, ph@entry=0x7ff961a31970) at include/jemalloc/internal/ph.h:349 #27 je_edata_heap_insert (ph=ph@entry=0x7ff961a14810, phn=phn@entry=0x7ff5c094cf80) at src/edata.c:6 #28 0x000000000048e617 in arena_bin_slabs_nonfull_insert (slab=0x7ff5c094cf80, bin=0x7ff961a14748) at src/arena.c:1339 #29 arena_bin_lower_slab (tsdn=0x7ff9635facb8, bin=0x7ff961a14748, slab=0x7ff5c094cf80, arena=0x7ff961a01040) at src/arena.c:1339 #30 je_arena_dalloc_bin_locked_handle_newly_nonempty (tsdn=tsdn@entry=0x7ff9635facb8, arena=arena@entry=0x7ff961a01040, slab=slab@entry=0x7ff5c094cf80, bin=bin@entry=0x7ff961a14748) at src/arena.c:1364 #31 0x00000000004ff336 in arena_dalloc_bin_locked_step (ptr=, slab=0x7ff5c094cf80, binind=, info=, bin=, arena=, tsdn=0x0) at include/jemalloc/internal/arena_inlines_b.h:528 #32 tcache_bin_flush_impl (small=true, nflush=, ptrs=0x7ff9635f8710, binind=, cache_bin=, tcache=, tsd=0x0) at src/tcache.c:448 #33 tcache_bin_flush_bottom (small=true, rem=, binind=1, cache_bin=, tcache=, tsd=tsd@entry=0x0) at src/tcache.c:519 #34 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9635facb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #35 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff4fd7abe30, tcache=, tsd=0x7ff9635facb8) at include/jemalloc/internal/tcache_inlines.h:157 #36 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #37 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #38 ifree (slow_path=false, tcache=0x7ff9635fb010, ptr=0x7ff4fd7abe30, tsd=0x7ff9635facb8) at src/jemalloc.c:2887 #39 je_free_default (ptr=0x7ff4fd7abe30) at src/jemalloc.c:3014 #40 0x00007ffeb65d6d64 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #41 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #42 0x000000000044f242 in stream_bridge_destroy_per_stream (mem_used_type=mem_used_type@entry=24, stream=stream@entry=0x7fffb3167fbc, pbridge=0x7ff5c6b22a04) at /home/yangwei/platform/sapp/src/project/stream_bridge.c:164 #43 0x000000000044452a in udp_free_stream (pindex=pindex@entry=0x7fffb3167f64) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:330 #44 0x0000000000446579 in del_stream (close_cause=6 '\006', pindex=0x7fffb3167f64) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:303 #45 streamaddlist (pindex=pindex@entry=0x7fffb28832c4, plist=plist@entry=0x7ffff36420b4) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:377 #46 0x0000000000443f39 in udp_add_new_stream (pindex=pindex@entry=0x7ff9635f8bc0, raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, this_udphdr=0x7f4226274afc) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:213 #47 0x0000000000444e35 in dealipv4udppkt (pindex=pindex@entry=0x7ff9635f8bc0, this_iphdr=this_iphdr@entry=0x7f4226274ae8, thread_num=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=108) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:667 #48 0x0000000000445b55 in process_ipv4_pkt (pfindex=pfindex@entry=0x7ff9635f8bc0, a_packet=a_packet@entry=0x7f4226274ae8, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=offset_to_raw_pkt_hdr@entry=88) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:1104 #49 0x000000000043780b in ipv4_entry (pfstream_pr=pfstream_pr@entry=0x7ff9635f8da8, this_layer_data=, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=) at /home/yangwei/platform/sapp/src/dealpkt/deal_ipv4.c:785 #50 0x000000000044a7cb in gre_entry (pfstream_pr=0x7ff9635f8fa8, pfstream_pr@entry=0x5d, this_layer_data=0x7f4226274ae4, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=84) at /home/yangwei/platform/sapp/src/dealpkt/deal_gre.c:282 #51 0x00000000004459bd in process_ipv4_pkt (pfindex=0x5, pfindex@entry=0x7ff9635f8f50, a_packet=a_packet@entry=0x7f4226274ad0, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=0, offset_to_raw_pkt_hdr@entry=64) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:1125 #52 0x000000000043780b in ipv4_entry (pfstream_pr=, this_layer_data=this_layer_data@entry=0x7f4226274ad0, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=) at /home/yangwei/platform/sapp/src/dealpkt/deal_ipv4.c:785 #53 0x00000000004436d4 in eth_entry (fstream_pr=fstream_pr@entry=0x7ff9635f92f8, this_layer_hdr=0x7f4226274ac2, thread_num=thread_num@entry=0, dir=dir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=) at /home/yangwei/platform/sapp/src/dealpkt/deal_ethernet.c:177 #54 0x000000000044cdd5 in vxlan_entry (pfstream_pr=pfstream_pr@entry=0x7ff9635f95e8, this_layer_hdr=this_layer_hdr@entry=0x7f4226274aba, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=offset_to_raw_pkt_hdr@entry=42) at /home/yangwei/platform/sapp/src/dealpkt/deal_vxlan.c:104 #55 0x0000000000445068 in dealipv4udppkt (pindex=pindex@entry=0x7ff9635f9590, this_iphdr=this_iphdr@entry=0x7f4226274a9e, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=34) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:758 #56 0x00000000004378dc in ipv4_entry (pfstream_pr=0x7ff9635f9798, this_layer_data=this_layer_data@entry=0x7f4226274a9e, thread_num=thread_num@entry=0, routedir=routedir@entry=0 '\000', raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=14) at /home/yangwei/platform/sapp/src/dealpkt/deal_ipv4.c:706 #57 0x00000000004436d4 in eth_entry (fstream_pr=fstream_pr@entry=0x0, this_layer_hdr=0x7f4226274a90, thread_num=thread_num@entry=0, dir=, raw_pkt=raw_pkt@entry=0x7ff9635fa2a0, offset_to_raw_pkt_hdr=offset_to_raw_pkt_hdr@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/deal_ethernet.c:177 #58 0x000000000042ed52 in mesa_default_pkt_cb (p_raw_pkt=0x7ff9635fa2a0, dir=, thread_num=) at /home/yangwei/platform/sapp/src/packet_io/packet_io.c:652 #59 0x0000000000508327 in marsio4_pkt_hand (dir=0 '\000', raw_pkt=0x7ff9635fa2a0, rx_buff=0x7f4226274900, tid=0) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:797 #60 marsio4_process_packet (tid=tid@entry=0, raw_pkt=raw_pkt@entry=0x7ff9635fa2a0) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:841 #61 0x0000000000508a31 in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:999 #62 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #63 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 44 (Thread 0x7ff962dfa700 (LWP 4036)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007ffb4280a356 in ?? () #4 0x00007fffb7f48e5c in ?? () #5 0x0000000000000001 in ?? () #6 0x00007ff962df8cd0 in ?? () #7 0x17aceabae9837ae0 in ?? () #8 0x17aceabae9837ae0 in ?? () #9 0x00007ffff73863da in fieldstat_value_operate(fieldstat_instance*, int, field_op, long long) () from /opt/tsg/framework/lib/libfieldstat3.so #10 0xffffffffffffffa8 in ?? () #11 0x00007ff962df8a90 in ?? () #12 0x00007ff942408d78 in ?? () #13 0xfffffffffffffff8 in ?? () #14 0x00007ffb4281b487 in ?? () #15 0x00007ff942440280 in ?? () #16 0x00007ff942434d00 in ?? () #17 0x00007ff942454a80 in ?? () #18 0x00007ff942448d00 in ?? () #19 0x00007ff942446f80 in ?? () #20 0x00007ff94240ef80 in ?? () #21 0x00007ff94241f000 in ?? () #22 0x00007ff942450880 in ?? () #23 0x00007ff942441080 in ?? () #24 0x00007ff94242ea00 in ?? () #25 0x000000000000003c in ?? () #26 0x0000000000000000 in ?? () Thread 43 (Thread 0x7ff9625f9700 (LWP 4037)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff95ea03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff95ea03a28) at src/mutex.c:90 #5 0x00000000004dd478 in malloc_mutex_lock (mutex=0x7ff95ea03a28, tsdn=0x7ff9625f8cb8) at include/jemalloc/internal/mutex.h:217 #6 extent_recycle (tsdn=tsdn@entry=0x7ff9625f8cb8, pac=pac@entry=0x7ff95ea039f0, ehooks=ehooks@entry=0x7ff95ea000c0, ecache=ecache@entry=0x7ff95ea03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=4096, alignment=4096, zero=false, commit=0x7ff9625f77df, growing_retained=false, guarded=false) at src/extent.c:604 #7 0x00000000004dd529 in je_ecache_alloc (tsdn=tsdn@entry=0x7ff9625f8cb8, pac=pac@entry=0x7ff95ea039f0, ehooks=ehooks@entry=0x7ff95ea000c0, ecache=ecache@entry=0x7ff95ea03a28, expand_edata=expand_edata@entry=0x0, size=size@entry=4096, alignment=4096, zero=false, guarded=false) at src/extent.c:87 #8 0x00000000004e890e in pac_alloc_real (tsdn=tsdn@entry=0x7ff9625f8cb8, pac=pac@entry=0x7ff95ea039f0, ehooks=ehooks@entry=0x7ff95ea000c0, size=size@entry=4096, alignment=alignment@entry=4096, zero=zero@entry=false, guarded=false) at src/pac.c:116 #9 0x00000000004e8aa5 in pac_alloc_impl (tsdn=0x7ff9625f8cb8, self=0x7ff95ea039f0, size=4096, alignment=4096, zero=, guarded=, frequent_reuse=true, deferred_work_generated=0x7ff9625f795f) at src/pac.c:178 #10 0x00000000004e7917 in pai_alloc (deferred_work_generated=0x7ff9625f795f, frequent_reuse=true, guarded=false, zero=false, alignment=, size=4096, self=0x7ff95ea039f0, tsdn=0x7ff9625f8cb8) at include/jemalloc/internal/pai.h:43 #11 je_pa_alloc (tsdn=tsdn@entry=0x7ff9625f8cb8, shard=shard@entry=0x7ff95ea039d8, size=4096, alignment=alignment@entry=4096, slab=slab@entry=true, szind=szind@entry=24, zero=false, guarded=false, deferred_work_generated=0x7ff9625f795f) at src/pa.c:139 #12 0x000000000048bbdd in arena_slab_alloc (tsdn=tsdn@entry=0x7ff9625f8cb8, arena=arena@entry=0x7ff95ea01040, binind=binind@entry=24, binshard=binshard@entry=0, bin_info=bin_info@entry=0x6e4c440 ) at src/arena.c:839 #13 0x000000000048d47b in je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ff9625f8cb8, arena=arena@entry=0x7ff95ea01040, cache_bin=cache_bin@entry=0x7ff9625f9258, cache_bin_info=, binind=binind@entry=24, nfill=) at src/arena.c:1034 #14 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ff9625f8cb8, arena=arena@entry=0x7ff95ea01040, tcache=tcache@entry=0x7ff9625f9010, cache_bin=cache_bin@entry=0x7ff9625f9258, binind=binind@entry=24, tcache_success=tcache_success@entry=0x7ff9625f7b10) at src/tcache.c:238 #15 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=24, size=, tcache=0x7ff9625f9010, arena=0x7ff95ea01040, tsd=) at include/jemalloc/internal/tsd.h:283 #16 arena_malloc (slow_path=, tcache=0x7ff9625f9010, zero=false, ind=24, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #17 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ff9625f9010, zero=false, ind=24, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #18 imalloc_no_sample (ind=24, usize=2048, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #19 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #20 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #21 je_malloc_default (size=) at src/jemalloc.c:2722 #22 0x000000000047e6ed in imalloc_fastpath (fallback_alloc=0x47cea0 , size=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:266 #23 malloc (size=) at src/jemalloc.c:2746 #24 0x00007ffeb706a83a in rd_malloc (sz=) at rd.h:127 #25 rd_kafka_msg_new00 (msg_opaque=0x0, keylen=0, key=0x0, len=1692, payload=0x7ff737f5f000 "{\"common_schema_type\":\"SSL\",\"common_sessions\":1,\"ssl_sni\":\"p16-amd-va.tiktokcdn.com\",\"ssl_con_latency_ms\":337,\"common_first_ttl\":56,\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_tcp_"..., msgflags=131074, partition=-1, rkt=0x7ffeb7c98000) at rdkafka_msg.c:160 #26 rd_kafka_msg_new0 (rkt=rkt@entry=0x7ffeb7c98000, force_partition=force_partition@entry=-1, msgflags=msgflags@entry=2, payload=payload@entry=0x7ff737f5f000 "{\"common_schema_type\":\"SSL\",\"common_sessions\":1,\"ssl_sni\":\"p16-amd-va.tiktokcdn.com\",\"ssl_con_latency_ms\":337,\"common_first_ttl\":56,\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_tcp_"..., len=len@entry=1692, key=key@entry=0x0, keylen=keylen@entry=0, msg_opaque=msg_opaque@entry=0x0, errp=errp@entry=0x7ff9625f7c4c, errnop=errnop@entry=0x7ff9625f7c50, hdrs=hdrs@entry=0x0, timestamp=timestamp@entry=0, now=9746784149837) at rdkafka_msg.c:257 #27 0x00007ffeb706bbef in rd_kafka_msg_new (rkt=0x7ffeb7c98000, force_partition=-1, msgflags=2, payload=0x7ff737f5f000 "{\"common_schema_type\":\"SSL\",\"common_sessions\":1,\"ssl_sni\":\"p16-amd-va.tiktokcdn.com\",\"ssl_con_latency_ms\":337,\"common_first_ttl\":56,\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_tcp_"..., len=1692, key=0x0, keylen=0, msg_opaque=0x0) at rdkafka_msg.c:315 #28 0x00007ffdf516861f in ?? () #29 0x0000000000000000 in ?? () Thread 42 (Thread 0x7ff9619ff700 (LWP 4038)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007ffb4280a356 in ?? () #4 0x746f72705f6e6f69 in ?? () #5 0x000000006c6f636f in ?? () #6 0x7376000000000074 in ?? () #7 0x17aceabae89369d8 in ?? () #8 0x17aceabae89369d8 in ?? () #9 0x6c5f6c6f636f746f in ?? () #10 0x485445006c656261 in ?? () #11 0x00007ff9619fd9a0 in ?? () #12 0x00007ff9619fdbd0 in ?? () #13 0xfffffffffffffcd8 in ?? () #14 0x00007ff9619fd680 in ?? () #15 0x00007ff945c08800 in ?? () #16 0xfffffffffffffff8 in ?? () #17 0x00007ffb4281b487 in ?? () #18 0x00007ff945c19a80 in ?? () #19 0x00007ff945c43b00 in ?? () #20 0x00007ff945c2f000 in ?? () #21 0x00007ff945c38980 in ?? () #22 0x00007ff945c31c80 in ?? () #23 0x00007ff945c35880 in ?? () #24 0x00007ff945c55480 in ?? () #25 0x00007ff945c2bc80 in ?? () #26 0x00007ff945c40800 in ?? () #27 0x00007ff945c32980 in ?? () #28 0x00007ff945c32980 in ?? () #29 0x00007ff945c39680 in ?? () #30 0x00007ff945c53400 in ?? () #31 0x00007ff945c38980 in ?? () #32 0x00007ff945c1c200 in ?? () #33 0x00007ff945c28f00 in ?? () #34 0x00007ff945c1c700 in ?? () #35 0x00007ff945c48a80 in ?? () #36 0x00007ff945c22f80 in ?? () #37 0x00007ff945c1c280 in ?? () #38 0x00007ff945c3e000 in ?? () #39 0x00007ff945c4d680 in ?? () #40 0x00007ff945c2f000 in ?? () #41 0x00007ff945c27e80 in ?? () #42 0x00007ff945c1c700 in ?? () #43 0x00007ff945c22f80 in ?? () #44 0x00007ff945c27d00 in ?? () #45 0x00007ff945c45180 in ?? () #46 0x00007ff945c1c280 in ?? () #47 0x00007ff945c3d480 in ?? () #48 0x00007ff945c20d80 in ?? () #49 0x00007ff945c22f80 in ?? () #50 0x00007ff945c54c80 in ?? () #51 0x00007ff945c39400 in ?? () #52 0x00007ff945c40800 in ?? () #53 0x00007ff945c32980 in ?? () #54 0x00007ff945c53400 in ?? () #55 0x00007ff945c4a880 in ?? () #56 0x00007ff945c17180 in ?? () #57 0x00007ff945c19b80 in ?? () #58 0x00007ff945c48a80 in ?? () #59 0x00007ff945c40800 in ?? () #60 0x00007ff945c35e80 in ?? () #61 0x00007ff945c2c600 in ?? () #62 0x00007ff945c36080 in ?? () #63 0x00007ff945c19d00 in ?? () #64 0x00007ff945c25980 in ?? () #65 0x00007ff945c19d00 in ?? () #66 0x00007ff945c27e80 in ?? () #67 0x00007ff945c35880 in ?? () #68 0x00007ff945c41080 in ?? () #69 0x00007ff945c2fe00 in ?? () #70 0x00007ff945c1c200 in ?? () #71 0x00007ff945c31c80 in ?? () #72 0x00007ff945c19180 in ?? () #73 0x00007ff945c4ed80 in ?? () #74 0x00007ff945c3a400 in ?? () #75 0x00007ff945c55480 in ?? () #76 0x00007ff945c1fe80 in ?? () #77 0x00007ff945c19a80 in ?? () #78 0x00007ff945c27d00 in ?? () #79 0x00007ff945c1c280 in ?? () #80 0x00007ff945c4ed80 in ?? () #81 0x00007ff945c1c280 in ?? () #82 0x00007ff945c48a80 in ?? () #83 0x00007ff945c4b780 in ?? () #84 0x00007ff945c2c600 in ?? () #85 0x00007ff945c2fe00 in ?? () #86 0x00007ff945c55480 in ?? () #87 0x00007ff945c40800 in ?? () #88 0x00007ff945c38c00 in ?? () #89 0x00007ff945c28f00 in ?? () #90 0x00007ff945c34d00 in ?? () #91 0x00007ff945c34d00 in ?? () #92 0x00007ff945c25980 in ?? () #93 0x00007ff945c2bc80 in ?? () #94 0x00007ff945c22280 in ?? () #95 0x00007ff945c42c00 in ?? () #96 0x00007ff945c35e80 in ?? () #97 0x00007ff945c44400 in ?? () #98 0x00007ff945c24600 in ?? () #99 0x00007ff945c25980 in ?? () #100 0x00007ff945c34d00 in ?? () #101 0x00007ff945c39680 in ?? () #102 0x00007ff945c19a80 in ?? () #103 0x00007ff945c34d00 in ?? () #104 0x00007ff945c22280 in ?? () #105 0x00007ff945c50500 in ?? () #106 0x00007ff945c50f00 in ?? () #107 0x00007ff945c43b00 in ?? () #108 0x00007ff945c6e180 in ?? () #109 0x00007ff945c22280 in ?? () #110 0x00007ff945c38c00 in ?? () #111 0x00007ff945c54c80 in ?? () #112 0x00007ff945c1f180 in ?? () #113 0x00007ff945c19b80 in ?? () #114 0x00007ff945c1fe80 in ?? () #115 0x00007ff945c1e780 in ?? () #116 0x00007ff945c48a80 in ?? () #117 0x00007ff945c32980 in ?? () #118 0x0000000000000000 in ?? () Thread 41 (Thread 0x7ff9611fe700 (LWP 4039)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffebc214748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffebc214748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffebc214748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff9611fc9c0, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3156297544, cache_bin=, tcache=, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9611fdcb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff7c2136d20, tcache=, tsd=0x7ff9611fdcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9611fe010, ptr=0x7ff7c2136d20, tsd=0x7ff9611fdcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff7c2136d20) at src/jemalloc.c:3014 #14 0x00007ffeb65d6ee4 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffdf515e3d8 in ?? () #17 0x0000000400000000 in ?? () #18 0x00000008611fcc30 in ?? () #19 0x00007ff9611fcb90 in ?? () #20 0x00007ff9611fcb90 in ?? () #21 0x00007ff3f3f48cf0 in ?? () #22 0x00007ffdf515b704 in ?? () #23 0x0000000000000001 in ?? () #24 0x0000000000000003 in ?? () #25 0x0000000000000000 in ?? () Thread 40 (Thread 0x7ff95ffff700 (LWP 4040)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffff3814608) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffff3814608) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffff3814608, tsdn=0x13548) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95fffdff0, binind=, cache_bin=, tcache=, tsd=0x13548) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4085335560, cache_bin=, tcache=, tsd=tsd@entry=0x13548) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95fffecb8, tcache=, cache_bin=, binind=binind@entry=1, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=1, ptr=0x7ffd9cb7e350, tcache=, tsd=0x7ff95fffecb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95ffff010, ptr=0x7ffd9cb7e350, tsd=0x7ff95fffecb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ffd9cb7e350) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PADDR, thread_seq=, data=) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448509 in free_heap_stream_info (mem_used_type=mem_used_type@entry=8, heap_stream=heap_stream@entry=0x7fff9dd3823c, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2156 #17 0x000000000043b3e7 in tcp_free_stream (pindex=0x7fff9dd381e4, this_ip_hdr=this_ip_hdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0) at /home/yangwei/platform/sapp/src/dealpkt/deal_tcp.c:1456 #18 0x0000000000446976 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=5) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:517 #19 0x0000000000449a1d in polling_stream_timeout (tid=tid@entry=5) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2734 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x7ff95f7fe700 (LWP 4041)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffebac14748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffebac14748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffebac14748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95f7fc390, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3133228872, cache_bin=, tcache=, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95f7fdcb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff311b35ec0, tcache=, tsd=0x7ff95f7fdcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95f7fe010, ptr=0x7ff311b35ec0, tsd=0x7ff95f7fdcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff311b35ec0) at src/jemalloc.c:3014 #14 0x00007ffac76e9bbe in ?? () #15 0x00007ff311b35ec0 in ?? () #16 0x0000000000000006 in ?? () #17 0x00007ff416c146c4 in ?? () #18 0x00007ff95f7fc640 in ?? () #19 0x00007fffa185b89c in ?? () #20 0x0000000600000000 in ?? () #21 0x0000000000000000 in ?? () Thread 38 (Thread 0x7ff95e9ff700 (LWP 4042)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffeb4e14828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffeb4e14828) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffeb4e14828, tsdn=0x137e8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=64, ptrs=0x7ff95e9fdfa0, binind=, cache_bin=, tcache=, tsd=0x137e8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3034662952, cache_bin=, tcache=, tsd=tsd@entry=0x137e8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95e9fecb8, tcache=, cache_bin=, binind=binind@entry=4, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=4, ptr=0x7ff87f280e00, tcache=, tsd=0x7ff95e9fecb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95e9ff010, ptr=0x7ff87f280e00, tsd=0x7ff95e9fecb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff87f280e00) at src/jemalloc.c:3014 #14 0x00007ffeb65d6dd1 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x000000000044f242 in stream_bridge_destroy_per_stream (mem_used_type=mem_used_type@entry=24, stream=stream@entry=0x7fff8844b8bc, pbridge=0x7ff90887e3c4) at /home/yangwei/platform/sapp/src/project/stream_bridge.c:164 #17 0x000000000044452a in udp_free_stream (pindex=0x7fff8844b864) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:330 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=7) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a9a in polling_stream_timeout (tid=tid@entry=7) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x7ff95dffe700 (LWP 4043)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffeb4c14668) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffeb4c14668) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffeb4c14668, tsdn=0x13628) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95dffd000, binind=, cache_bin=, tcache=, tsd=0x13628) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3032565352, cache_bin=, tcache=, tsd=tsd@entry=0x13628) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95dffdcb8, tcache=, cache_bin=, binind=binind@entry=2, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=2, ptr=0x7ff4c18c76c0, tcache=, tsd=0x7ff95dffdcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95dffe010, ptr=0x7ff4c18c76c0, tsd=0x7ff95dffdcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff4c18c76c0) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PADDR, thread_seq=, data=) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448509 in free_heap_stream_info (mem_used_type=mem_used_type@entry=9, heap_stream=heap_stream@entry=0x7fff8ce60edc, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2156 #17 0x00000000004445d9 in udp_free_stream (pindex=0x7fff8ce60e84) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:358 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=8) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=8) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x7ff95d7fd700 (LWP 4044)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffeb3c14e48) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffeb3c14e48) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffeb3c14e48, tsdn=0x13e08) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95d7fc000, binind=, cache_bin=, tcache=, tsd=0x13e08) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3015790152, cache_bin=, tcache=, tsd=tsd@entry=0x13e08) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95d7fccb8, tcache=, cache_bin=, binind=binind@entry=11, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=11, ptr=0x7ff9396492a0, tcache=, tsd=0x7ff95d7fccb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95d7fd010, ptr=0x7ff9396492a0, tsd=0x7ff95d7fccb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff9396492a0) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_UDP_STREAM, thread_seq=, data=data@entry=0x7ff9396492a4) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448529 in free_heap_stream_info (mem_used_type=mem_used_type@entry=9, heap_stream=heap_stream@entry=0x7fff8ec96ffc, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2165 #17 0x00000000004445d9 in udp_free_stream (pindex=0x7fff8ec96fa4) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:358 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=9) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=9) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x7ff95cffc700 (LWP 4045)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf7a14ba8) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf7a14ba8) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffdf7a14ba8, tsdn=0x13b68) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=32, ptrs=0x7ff95cffaa30, binind=, cache_bin=, tcache=, tsd=0x13b68) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4154543016, cache_bin=, tcache=, tsd=tsd@entry=0x13b68) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95cffbcb8, tcache=, cache_bin=, binind=binind@entry=8, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=8, ptr=0x7ff7a55e1c00, tcache=, tsd=0x7ff95cffbcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95cffc010, ptr=0x7ff7a55e1c00, tsd=0x7ff95cffbcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff7a55e1c00) at src/jemalloc.c:3014 #14 0x00007ffeb65d6dd1 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffdf515e3d8 in ?? () #17 0x0000000a00000000 in ?? () #18 0x0000000000000000 in ?? () Thread 34 (Thread 0x7ff95c7fb700 (LWP 4046)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf6c14e48) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf6c14e48) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffdf6c14e48, tsdn=0x13e08) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95c7f9ff0, binind=, cache_bin=, tcache=, tsd=0x13e08) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4139863624, cache_bin=, tcache=, tsd=tsd@entry=0x13e08) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95c7facb8, tcache=, cache_bin=, binind=binind@entry=11, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=11, ptr=0x7fee3928b3e0, tcache=, tsd=0x7ff95c7facb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95c7fb010, ptr=0x7fee3928b3e0, tsd=0x7ff95c7facb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7fee3928b3e0) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_TCP_STREAM, thread_seq=, data=data@entry=0x7fee3928b3e4) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448529 in free_heap_stream_info (mem_used_type=mem_used_type@entry=8, heap_stream=heap_stream@entry=0x7fff7827b9bc, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2165 #17 0x000000000043b3e7 in tcp_free_stream (pindex=0x7fff7827b964, this_ip_hdr=this_ip_hdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0) at /home/yangwei/platform/sapp/src/dealpkt/deal_tcp.c:1456 #18 0x0000000000446976 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=11) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:517 #19 0x0000000000449a1d in polling_stream_timeout (tid=tid@entry=11) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2734 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x7ff95bffa700 (LWP 4047)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffdf5614748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffdf5614748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffdf5614748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95bff8a30, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4116793160, cache_bin=, tcache=, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95bff9cb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff3ccf4f8d0, tcache=, tsd=0x7ff95bff9cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95bffa010, ptr=0x7ff3ccf4f8d0, tsd=0x7ff95bff9cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff3ccf4f8d0) at src/jemalloc.c:3014 #14 0x00007ffeb65d6ee4 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffdf515e3d8 in ?? () #17 0x0000000c00000000 in ?? () #18 0x0000000000000000 in ?? () Thread 32 (Thread 0x7ff95b7f9700 (LWP 4048)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb47e152a8) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb47e152a8) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffb47e152a8, tsdn=0x14268) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff95b7f7c80, binind=, cache_bin=, tcache=, tsd=0x14268) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1205949096, cache_bin=, tcache=, tsd=tsd@entry=0x14268) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95b7f8cb8, tcache=, cache_bin=, binind=binind@entry=16, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=16, ptr=0x7ff6801e4800, tcache=, tsd=0x7ff95b7f8cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95b7f9010, ptr=0x7ff6801e4800, tsd=0x7ff95b7f8cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff6801e4800) at src/jemalloc.c:3014 #14 0x00007ffacb262849 in ?? () #15 0x0000000000040005 in ?? () #16 0x0000000000000002 in ?? () #17 0x0000000000000000 in ?? () Thread 31 (Thread 0x7ff95aff8700 (LWP 4049)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb46014828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb46014828) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffb46014828, tsdn=0x137e8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=64, ptrs=0x7ff95aff6a30, binind=, cache_bin=, tcache=, tsd=0x137e8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1174489128, cache_bin=, tcache=, tsd=tsd@entry=0x137e8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95aff7cb8, tcache=, cache_bin=, binind=binind@entry=4, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=4, ptr=0x7ff4e645e280, tcache=, tsd=0x7ff95aff7cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95aff8010, ptr=0x7ff4e645e280, tsd=0x7ff95aff7cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff4e645e280) at src/jemalloc.c:3014 #14 0x00007ffeb65d6ef4 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffdf515e3d8 in ?? () #17 0x0000000e00000000 in ?? () #18 0x0000000000000000 in ?? () Thread 30 (Thread 0x7ff95a7f7700 (LWP 4050)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb45c14c88) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb45c14c88) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffb45c14c88, tsdn=0x13c48) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff95a7f5c40, binind=, cache_bin=, tcache=, tsd=0x13c48) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1170295944, cache_bin=, tcache=, tsd=tsd@entry=0x13c48) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff95a7f6cb8, tcache=, cache_bin=, binind=binind@entry=9, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=9, ptr=0x7ff813058580, tcache=, tsd=0x7ff95a7f6cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff95a7f7010, ptr=0x7ff813058580, tsd=0x7ff95a7f6cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff813058580) at src/jemalloc.c:3014 #14 0x00007ffb433f2e36 in ?? () #15 0x000000000000000f in ?? () #16 0x00007fff5de30d7c in ?? () #17 0x00007ff813058580 in ?? () #18 0x00007ffb433f0621 in ?? () #19 0x0000000000000000 in ?? () Thread 29 (Thread 0x7ff959ff6700 (LWP 4051)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffb45815e08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffb45815e08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffb45815e08, tsdn=0x14dc8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff959ff4c50, binind=, cache_bin=, tcache=, tsd=0x14dc8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1166106120, cache_bin=, tcache=, tsd=tsd@entry=0x14dc8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff959ff5cb8, tcache=, cache_bin=, binind=binind@entry=29, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=29, ptr=0x7fefda2a5800, tcache=, tsd=0x7ff959ff5cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff959ff6010, ptr=0x7fefda2a5800, tsd=0x7ff959ff5cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7fefda2a5800) at src/jemalloc.c:3014 #14 0x00007ffaafc008a0 in ?? () #15 0x00007fff64c8d6bc in ?? () #16 0x00007ffaafc04913 in ?? () #17 0x0000000000000001 in ?? () #18 0x0000000000000010 in ?? () #19 0x0000000000000002 in ?? () #20 0x00007ffaafe199cf in ?? () #21 0x0000000000000010 in ?? () #22 0x0000000000000002 in ?? () #23 0x0000000000000000 in ?? () Thread 28 (Thread 0x7ff9597f5700 (LWP 4063)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7fface614e48) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7fface614e48) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7fface614e48, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=65, ptrs=0x7ff9597f3b60, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=, tcache=, cache_bin=, binind=, rem=) at src/tcache.c:529 #9 0x0000000000500326 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:430 #10 0x0000000000502c49 in je_te_event_trigger (tsd=0x7ff9597f4cb8, ctx=) at src/thread_event.c:299 #11 0x0000000000481a60 in te_event_advance (is_alloc=, usize=, tsd=) at include/jemalloc/internal/thread_event.h:287 #12 thread_dalloc_event (usize=, tsd=) at include/jemalloc/internal/thread_event.h:293 #13 ifree (slow_path=, tcache=, ptr=, tsd=) at src/jemalloc.c:2896 #14 je_free_default (ptr=) at src/jemalloc.c:3021 #15 0x00007ffdf51671d3 in ?? () #16 0x00007ff44954b9a0 in ?? () #17 0x00007fffec1fe800 in ?? () #18 0x00007ff44954b9a0 in ?? () #19 0x00007fff656f0d7c in ?? () #20 0x0000000000000001 in ?? () #21 0x00007ffdf51710fa in ?? () #22 0x0000000000000011 in ?? () #23 0x000000000000004a in ?? () #24 0x00007ff9597f3de0 in ?? () #25 0x0000000000000001 in ?? () #26 0x00007ff44954b9a0 in ?? () #27 0x00000000656f0d7c in ?? () #28 0x00007fff656f0d7c in ?? () #29 0x00007ff44954b9a0 in ?? () #30 0x0000000000000001 in ?? () #31 0x0000000000000011 in ?? () #32 0x0000000000000002 in ?? () #33 0x00007ffaafe199cf in ?? () #34 0x0000000000000011 in ?? () #35 0x0000000000000002 in ?? () #36 0x0000000000000000 in ?? () Thread 27 (Thread 0x7ff958ff4700 (LWP 4064)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffac8814668) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffac8814668) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffac8814668, tsdn=0x13628) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff958ff3000, binind=, cache_bin=, tcache=, tsd=0x13628) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3363915368, cache_bin=, tcache=, tsd=tsd@entry=0x13628) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff958ff3cb8, tcache=, cache_bin=, binind=binind@entry=2, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=2, ptr=0x7ff5e6fe1280, tcache=, tsd=0x7ff958ff3cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff958ff4010, ptr=0x7ff5e6fe1280, tsd=0x7ff958ff3cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff5e6fe1280) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PADDR, thread_seq=, data=) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448509 in free_heap_stream_info (mem_used_type=mem_used_type@entry=9, heap_stream=heap_stream@entry=0x7fff69ffd59c, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2156 #17 0x00000000004445d9 in udp_free_stream (pindex=0x7fff69ffd544) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:358 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=18) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=18) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7ff9587f3700 (LWP 4065)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaabe14828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaabe14828) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaabe14828, tsdn=0x137e8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=64, ptrs=0x7ff9587f19c0, binind=, cache_bin=, tcache=, tsd=0x137e8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2883668008, cache_bin=, tcache=, tsd=tsd@entry=0x137e8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9587f2cb8, tcache=, cache_bin=, binind=binind@entry=4, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=4, ptr=0x7ff5f288ffc0, tcache=, tsd=0x7ff9587f2cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9587f3010, ptr=0x7ff5f288ffc0, tsd=0x7ff9587f2cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff5f288ffc0) at src/jemalloc.c:3014 #14 0x00007ffeb65d6dd1 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffdf515e3d8 in ?? () #17 0x0000001300000000 in ?? () #18 0x00000008587f1c30 in ?? () #19 0x00007ff9587f1b90 in ?? () #20 0x00007ff9587f1b90 in ?? () #21 0x00007ff480baf250 in ?? () #22 0x00007ffdf515b704 in ?? () #23 0x0000000000000001 in ?? () #24 0x0000000000000001 in ?? () #25 0x0000000000000000 in ?? () Thread 25 (Thread 0x7ff957ff2700 (LWP 4066)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaae415a88) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaae415a88) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaae415a88, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=3, ptrs=0x7ff957ff0b90, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=, tcache=, cache_bin=, binind=, rem=) at src/tcache.c:529 #9 0x0000000000500326 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:430 #10 0x0000000000502c49 in je_te_event_trigger (tsd=0x7ff957ff1cb8, ctx=) at src/thread_event.c:299 #11 0x0000000000481a60 in te_event_advance (is_alloc=, usize=, tsd=) at include/jemalloc/internal/thread_event.h:287 #12 thread_dalloc_event (usize=, tsd=) at include/jemalloc/internal/thread_event.h:293 #13 ifree (slow_path=, tcache=, ptr=, tsd=) at src/jemalloc.c:2896 #14 je_free_default (ptr=) at src/jemalloc.c:3021 #15 0x00007ffdf51671d3 in ?? () #16 0x00007fffec1fe800 in ?? () #17 0x00007fffec1fe800 in ?? () #18 0x00007ff492c88c20 in ?? () #19 0x00007fff42c12a5c in ?? () #20 0x0000000000000001 in ?? () #21 0x00007ffdf51710fa in ?? () #22 0x00007ff500000014 in ?? () #23 0x0000000000000002 in ?? () #24 0x00007ff957ff0e10 in ?? () #25 0x0000000000000001 in ?? () #26 0x00007ff492c88c20 in ?? () #27 0x0000000142c12a5c in ?? () #28 0x00007fff42c12a5c in ?? () #29 0x00007ff492c88c20 in ?? () #30 0x0000000000000001 in ?? () #31 0x0000000000000000 in ?? () Thread 24 (Thread 0x7ff9577f1700 (LWP 4067)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaaac14c88) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaaac14c88) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaaac14c88, tsdn=0x13c48) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff9577efea0, binind=, cache_bin=, tcache=, tsd=0x13c48) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2864794760, cache_bin=, tcache=, tsd=tsd@entry=0x13c48) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9577f0cb8, tcache=, cache_bin=, binind=binind@entry=9, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=9, ptr=0x7ff6855d52a0, tcache=, tsd=0x7ff9577f0cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9577f1010, ptr=0x7ff6855d52a0, tsd=0x7ff9577f0cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff6855d52a0) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PLUG_CTRL, thread_seq=, data=data@entry=0x7ff6855d52a4) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x000000000043630a in stream_process (a_stream=a_stream@entry=0x7fff46643c7c, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, funnum=19, pfunAarry=pfunAarry@entry=0x9946c0 , apme=0x7ff51dccbc74, opstate=0x7fff46643c98 "\002\002\001", smart_offload_flag=1 '\001') at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1022 #17 0x00000000004368a7 in stream_process_udp (a_udp=a_udp@entry=0x7fff46643c7c, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, apme=apme@entry=0x7ff51dccbc74, popstate=popstate@entry=0x7fff46643c98 "\002\002\001") at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1125 #18 0x00000000004444cd in udp_free_stream (pindex=0x7fff46643c24) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:323 #19 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=21) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #20 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=21) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #21 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #22 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7ff956ff0700 (LWP 4068)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa9814828) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa9814828) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa9814828, tsdn=0x137e8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=64, ptrs=0x7ff956feefa0, binind=, cache_bin=, tcache=, tsd=0x137e8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2843822120, cache_bin=, tcache=, tsd=tsd@entry=0x137e8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff956fefcb8, tcache=, cache_bin=, binind=binind@entry=4, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=4, ptr=0x7ff6a20d0140, tcache=, tsd=0x7ff956fefcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff956ff0010, ptr=0x7ff6a20d0140, tsd=0x7ff956fefcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff6a20d0140) at src/jemalloc.c:3014 #14 0x00007ffeb65d6dd1 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x000000000044f242 in stream_bridge_destroy_per_stream (mem_used_type=mem_used_type@entry=24, stream=stream@entry=0x7fff4a794c9c, pbridge=0x7ff48bd2d444) at /home/yangwei/platform/sapp/src/project/stream_bridge.c:164 #17 0x000000000044452a in udp_free_stream (pindex=0x7fff4a794c44) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:330 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=22) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=22) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7ff9567ef700 (LWP 4069)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa9414748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa9414748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa9414748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff9567edfe0, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2839627592, cache_bin=, tcache=, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9567eecb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff3c8766470, tcache=, tsd=0x7ff9567eecb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9567ef010, ptr=0x7ff3c8766470, tsd=0x7ff9567eecb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff3c8766470) at src/jemalloc.c:3014 #14 0x000000000044f242 in stream_bridge_destroy_per_stream (mem_used_type=mem_used_type@entry=23, stream=stream@entry=0x7fff4e0cd4bc, pbridge=0x7fef5fb88084) at /home/yangwei/platform/sapp/src/project/stream_bridge.c:164 #15 0x000000000043b292 in tcp_free_stream (pindex=0x7fff4e0cd464, this_ip_hdr=this_ip_hdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0) at /home/yangwei/platform/sapp/src/dealpkt/deal_tcp.c:1426 #16 0x0000000000446976 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=23) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:517 #17 0x0000000000449a1d in polling_stream_timeout (tid=tid@entry=23) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2734 #18 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #19 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #20 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7ff955fee700 (LWP 4070)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa9014588) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa9014588) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa9014588, tsdn=0x13548) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff955fecc40, binind=, cache_bin=, tcache=, tsd=0x13548) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2835432840, cache_bin=, tcache=, tsd=tsd@entry=0x13548) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff955fedcb8, tcache=, cache_bin=, binind=binind@entry=1, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=1, ptr=0x7ff263d43f40, tcache=, tsd=0x7ff955fedcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff955fee010, ptr=0x7ff263d43f40, tsd=0x7ff955fedcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff263d43f40) at src/jemalloc.c:3014 #14 0x00007ffac76ef151 in ?? () #15 0x00007ff955fecdbf in ?? () #16 0x00007ff800000018 in ?? () #17 0x0000000000000000 in ?? () Thread 20 (Thread 0x7ff9557ed700 (LWP 4071)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa8c14748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa8c14748) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa8c14748, tsdn=0x13708) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff9557ebfa0, binind=, cache_bin=, tcache=, tsd=0x13708) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2831238984, cache_bin=, tcache=, tsd=tsd@entry=0x13708) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9557eccb8, tcache=, cache_bin=, binind=binind@entry=3, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=3, ptr=0x7ff81e395be0, tcache=, tsd=0x7ff9557eccb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9557ed010, ptr=0x7ff81e395be0, tsd=0x7ff9557eccb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff81e395be0) at src/jemalloc.c:3014 #14 0x00007ffeb65d6ce4 in maat_compile_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #15 0x00007ffeb65c543a in maat_state_free () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x000000000044f242 in stream_bridge_destroy_per_stream (mem_used_type=mem_used_type@entry=24, stream=stream@entry=0x7fff1bd58c3c, pbridge=0x7ff1bfa678c4) at /home/yangwei/platform/sapp/src/project/stream_bridge.c:164 #17 0x000000000044452a in udp_free_stream (pindex=0x7fff1bd58be4) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:330 #18 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=25) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #19 0x0000000000449a70 in polling_stream_timeout (tid=tid@entry=25) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7ff954fec700 (LWP 4072)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa8815e08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa8815e08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa8815e08, tsdn=0x14dc8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff954feac50, binind=, cache_bin=, tcache=, tsd=0x14dc8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2827050504, cache_bin=, tcache=, tsd=tsd@entry=0x14dc8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff954febcb8, tcache=, cache_bin=, binind=binind@entry=29, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=29, ptr=0x7ff5d5e66800, tcache=, tsd=0x7ff954febcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff954fec010, ptr=0x7ff5d5e66800, tsd=0x7ff954febcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff5d5e66800) at src/jemalloc.c:3014 #14 0x00007ffaafc008a0 in ?? () #15 0x00007fff21fb43fc in ?? () #16 0x00007ffaafc04913 in ?? () #17 0x0000000000000001 in ?? () #18 0x000000000000001a in ?? () #19 0x0000000000000002 in ?? () #20 0x00007ffaafe199cf in ?? () #21 0x000000000000001a in ?? () #22 0x0000000000000002 in ?? () #23 0x0000000000000000 in ?? () Thread 18 (Thread 0x7ff9547eb700 (LWP 4073)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa84157e8) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa84157e8) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ff9547eacb8, arena=arena@entry=0x7ffaa8401040, cache_bin=cache_bin@entry=0x7ff9547eb228, cache_bin_info=, binind=binind@entry=22, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ff9547eacb8, arena=arena@entry=0x7ffaa8401040, tcache=tcache@entry=0x7ff9547eb010, cache_bin=cache_bin@entry=0x7ff9547eb228, binind=binind@entry=22, tcache_success=tcache_success@entry=0x7ff9547e9af0) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=22, size=, tcache=0x7ff9547eb010, arena=0x7ffaa8401040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ff9547eb010, zero=false, ind=22, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ff9547eb010, zero=false, ind=22, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=22, usize=1536, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x000000000047e6ed in imalloc_fastpath (fallback_alloc=0x47cea0 , size=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:266 #16 malloc (size=) at src/jemalloc.c:2746 #17 0x00007ffeb706a83a in rd_malloc (sz=) at rd.h:127 #18 rd_kafka_msg_new00 (msg_opaque=0x0, keylen=0, key=0x0, len=1229, payload=0x7ff99bf46000 "{\"common_schema_type\":\"QUIC\",\"common_sessions\":1,\"quic_version\":\"IETF QUIC 29\",\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_byte_diff\":0,\"common_c2s_pkt_diff\":0,\"common_s2c_byte_dif"..., msgflags=131074, partition=-1, rkt=0x7ffeb7c98000) at rdkafka_msg.c:160 #19 rd_kafka_msg_new0 (rkt=rkt@entry=0x7ffeb7c98000, force_partition=force_partition@entry=-1, msgflags=msgflags@entry=2, payload=payload@entry=0x7ff99bf46000 "{\"common_schema_type\":\"QUIC\",\"common_sessions\":1,\"quic_version\":\"IETF QUIC 29\",\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_byte_diff\":0,\"common_c2s_pkt_diff\":0,\"common_s2c_byte_dif"..., len=len@entry=1229, key=key@entry=0x0, keylen=keylen@entry=0, msg_opaque=msg_opaque@entry=0x0, errp=errp@entry=0x7ff9547e9c2c, errnop=errnop@entry=0x7ff9547e9c30, hdrs=hdrs@entry=0x0, timestamp=timestamp@entry=0, now=9746784123856) at rdkafka_msg.c:257 #20 0x00007ffeb706bbef in rd_kafka_msg_new (rkt=0x7ffeb7c98000, force_partition=-1, msgflags=2, payload=0x7ff99bf46000 "{\"common_schema_type\":\"QUIC\",\"common_sessions\":1,\"quic_version\":\"IETF QUIC 29\",\"common_c2s_ipfrag_num\":0,\"common_s2c_ipfrag_num\":0,\"common_c2s_byte_diff\":0,\"common_c2s_pkt_diff\":0,\"common_s2c_byte_dif"..., len=1229, key=0x0, keylen=0, msg_opaque=0x0) at rdkafka_msg.c:315 #21 0x00007ffdf516861f in ?? () #22 0x0000000000000000 in ?? () Thread 17 (Thread 0x7ff953fea700 (LWP 4074)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffaa2415e08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffaa2415e08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffaa2415e08, tsdn=0x14dc8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff953fe8c50, binind=, cache_bin=, tcache=, tsd=0x14dc8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2722192904, cache_bin=, tcache=, tsd=tsd@entry=0x14dc8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff953fe9cb8, tcache=, cache_bin=, binind=binind@entry=29, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=29, ptr=0x7ff664a06400, tcache=, tsd=0x7ff953fe9cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff953fea010, ptr=0x7ff664a06400, tsd=0x7ff953fe9cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff664a06400) at src/jemalloc.c:3014 #14 0x00007ffaafc008a0 in ?? () #15 0x00007fff2847fc9c in ?? () #16 0x00007ffaafc04913 in ?? () #17 0x0000000000000001 in ?? () #18 0x000000000000001c in ?? () #19 0x0000000000000002 in ?? () #20 0x00007ffaafe199cf in ?? () #21 0x000000000000001c in ?? () #22 0x0000000000000002 in ?? () #23 0x0000000000000000 in ?? () Thread 16 (Thread 0x7ff9537e9700 (LWP 4077)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa9b614908) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa9b614908) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffa9b614908, tsdn=0x138c8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=31, ptrs=0x7ff9537e7400, binind=, cache_bin=, tcache=, tsd=0x138c8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2606844168, cache_bin=0x7ff9537e7440, tcache=0x1a, tsd=tsd@entry=0x138c8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9537e8cb8, tcache=tcache@entry=0x7ff9537e9010, cache_bin=cache_bin@entry=0x7ff9537e9090, binind=binind@entry=5, rem=) at src/tcache.c:529 #9 0x00000000005004e6 in tcache_gc_small (szind=5, tcache=0x7ff9537e9010, tcache_slow=0x7ff9537e8db8, tsd=0x7ff9537e8cb8) at src/tcache.c:148 #10 tcache_event (tsd=tsd@entry=0x7ff9537e8cb8) at src/tcache.c:193 #11 je_tcache_gc_event_handler (tsd=tsd@entry=0x7ff9537e8cb8, elapsed=elapsed@entry=18446744073709551615) at src/tcache.c:219 #12 0x0000000000502d9b in je_te_event_trigger (tsd=0x7ff9537e8cb8, ctx=) at src/thread_event.c:299 #13 0x000000000047d4b1 in te_event_advance (is_alloc=true, usize=5120, tsd=) at include/jemalloc/internal/thread_event.h:287 #14 thread_alloc_event (usize=5120, tsd=) at include/jemalloc/internal/thread_event.h:298 #15 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2585 #16 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #17 je_malloc_default (size=) at src/jemalloc.c:2722 #18 0x00007ffdf51724b6 in ?? () #19 0x0000000000001000 in ?? () #20 0x00007ff52f932620 in ?? () #21 0x00007ff9537e7600 in ?? () #22 0x00007ff9537e7940 in ?? () #23 0x0000000000000012 in ?? () #24 0x00007ffa9a6a5620 in ?? () #25 0x00007ff9537e7620 in ?? () #26 0x00007ffa98a273b0 in ?? () #27 0x00007ff9537e7630 in ?? () #28 0x00007ffdf516895b in ?? () #29 0x0000000000000000 in ?? () Thread 15 (Thread 0x7ff952fe8700 (LWP 4079)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa95014c88) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa95014c88) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffa95014c88, tsdn=0x13c48) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff952fe6e90, binind=, cache_bin=, tcache=, tsd=0x13c48) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2499890312, cache_bin=, tcache=, tsd=tsd@entry=0x13c48) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff952fe7cb8, tcache=, cache_bin=, binind=binind@entry=9, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=9, ptr=0x7ffa7780cf80, tcache=, tsd=0x7ff952fe7cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff952fe8010, ptr=0x7ffa7780cf80, tsd=0x7ff952fe7cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ffa7780cf80) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PLUG_CTRL, thread_seq=, data=data@entry=0x7ffa7780cf84) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x000000000043630a in stream_process (a_stream=a_stream@entry=0x7fff313d94fc, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, funnum=6, pfunAarry=pfunAarry@entry=0x994ae0 , apme=0x7ffa68107ef4, opstate=0x7fff313d9519 "\002", smart_offload_flag=2 '\002') at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1022 #17 0x0000000000436957 in stream_process_tcp_allpkt (a_tcp=a_tcp@entry=0x7fff313d94fc, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, apme=apme@entry=0x7ffa68107ef4, popstate=popstate@entry=0x7fff313d9519 "\002") at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1171 #18 0x000000000043b272 in tcp_free_stream (pindex=0x7fff313d94a4, this_ip_hdr=this_ip_hdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0) at /home/yangwei/platform/sapp/src/dealpkt/deal_tcp.c:1421 #19 0x0000000000446976 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=30) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:517 #20 0x0000000000449a1d in polling_stream_timeout (tid=tid@entry=30) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2734 #21 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #22 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7ff9527e7700 (LWP 4084)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffa8e215e08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffa8e215e08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffa8e215e08, tsdn=0x14dc8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff9527e5c50, binind=, cache_bin=, tcache=, tsd=0x14dc8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=2384551432, cache_bin=, tcache=, tsd=tsd@entry=0x14dc8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff9527e6cb8, tcache=, cache_bin=, binind=binind@entry=29, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=29, ptr=0x7ffa5e7f0c00, tcache=, tsd=0x7ff9527e6cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff9527e7010, ptr=0x7ffa5e7f0c00, tsd=0x7ff9527e6cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ffa5e7f0c00) at src/jemalloc.c:3014 #14 0x00007ffaafc008a0 in ?? () #15 0x00007fff350f39bc in ?? () #16 0x00007ffaafc04913 in ?? () #17 0x0000000000000001 in ?? () #18 0x000000000000001f in ?? () #19 0x0000000000000002 in ?? () #20 0x00007ffaafe199cf in ?? () #21 0x000000000000001f in ?? () #22 0x0000000000000002 in ?? () #23 0x0000000000000000 in ?? () Thread 13 (Thread 0x7ff951fe6700 (LWP 4085)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff966a159a8) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff966a159a8) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ff966a159a8, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=1, ptrs=0x7ff951fe4b90, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=, tcache=, cache_bin=, binind=, rem=) at src/tcache.c:529 #9 0x0000000000500326 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:430 #10 0x0000000000502c49 in je_te_event_trigger (tsd=0x7ff951fe5cb8, ctx=) at src/thread_event.c:299 #11 0x0000000000481a60 in te_event_advance (is_alloc=, usize=, tsd=) at include/jemalloc/internal/thread_event.h:287 #12 thread_dalloc_event (usize=, tsd=) at include/jemalloc/internal/thread_event.h:293 #13 ifree (slow_path=, tcache=, ptr=, tsd=) at src/jemalloc.c:2896 #14 je_free_default (ptr=) at src/jemalloc.c:3021 #15 0x00007ffdf51671d3 in ?? () #16 0x00007ff39e6caaa0 in ?? () #17 0x00007fffec1fe800 in ?? () #18 0x00007ff39e6caaa0 in ?? () #19 0x00007ffef21259dc in ?? () #20 0x0000000000000001 in ?? () #21 0x00007ffdf51710fa in ?? () #22 0x00007ff200000020 in ?? () #23 0x0000000000000000 in ?? () Thread 12 (Thread 0x7ff9517e5700 (LWP 4088)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff965c152a8) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff965c152a8) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ff965c152a8, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=3, ptrs=0x7ff9517e3bc0, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=, tcache=, cache_bin=, binind=, rem=) at src/tcache.c:529 #9 0x0000000000500326 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:430 #10 0x0000000000502c49 in je_te_event_trigger (tsd=0x7ff9517e4cb8, ctx=) at src/thread_event.c:299 #11 0x0000000000481a60 in te_event_advance (is_alloc=, usize=, tsd=) at include/jemalloc/internal/thread_event.h:287 #12 thread_dalloc_event (usize=, tsd=) at include/jemalloc/internal/thread_event.h:293 #13 ifree (slow_path=, tcache=, ptr=, tsd=) at src/jemalloc.c:2896 #14 je_free_default (ptr=) at src/jemalloc.c:3021 #15 0x00007ffaafc008a0 in ?? () #16 0x00007ffef77b30fc in ?? () #17 0x00007ffaafc04913 in ?? () #18 0x0000000000000001 in ?? () #19 0x0000000000000021 in ?? () #20 0x0000000000000002 in ?? () #21 0x00007ffaafe199cf in ?? () #22 0x0000000000000021 in ?? () #23 0x0000000000000002 in ?? () #24 0x0000000000000000 in ?? () Thread 11 (Thread 0x7ff950fe4700 (LWP 4089)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff965814588) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff965814588) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ff965814588, tsdn=0x13548) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff950fe2ff0, binind=, cache_bin=, tcache=, tsd=0x13548) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=1702970760, cache_bin=, tcache=, tsd=tsd@entry=0x13548) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff950fe3cb8, tcache=, cache_bin=, binind=binind@entry=1, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=1, ptr=0x7ff8f3bdb4e0, tcache=, tsd=0x7ff950fe3cb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff950fe4010, ptr=0x7ff8f3bdb4e0, tsd=0x7ff950fe3cb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff8f3bdb4e0) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PADDR, thread_seq=, data=) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x0000000000448509 in free_heap_stream_info (mem_used_type=mem_used_type@entry=8, heap_stream=heap_stream@entry=0x7ffef9af00bc, layer=layer@entry=0) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2156 #17 0x000000000043b3e7 in tcp_free_stream (pindex=0x7ffef9af0064, this_ip_hdr=this_ip_hdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0) at /home/yangwei/platform/sapp/src/dealpkt/deal_tcp.c:1456 #18 0x0000000000446976 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=34) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:517 #19 0x0000000000449a1d in polling_stream_timeout (tid=tid@entry=34) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2734 #20 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #21 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #22 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7ff9507e3700 (LWP 4090)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff961a14748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff961a14748) at src/mutex.c:90 #5 0x000000000048d49a in malloc_mutex_lock (mutex=, tsdn=) at include/jemalloc/internal/mutex.h:217 #6 je_arena_cache_bin_fill_small (tsdn=tsdn@entry=0x7ff9507e2cb8, arena=arena@entry=0x7ff961a01040, cache_bin=cache_bin@entry=0x7ff9507e3060, cache_bin_info=, binind=binind@entry=3, nfill=) at src/arena.c:972 #7 0x00000000004fdfa9 in je_tcache_alloc_small_hard (tsdn=tsdn@entry=0x7ff9507e2cb8, arena=arena@entry=0x7ff961a01040, tcache=tcache@entry=0x7ff9507e3010, cache_bin=cache_bin@entry=0x7ff9507e3060, binind=binind@entry=3, tcache_success=tcache_success@entry=0x7ff9507afd00) at src/tcache.c:238 #8 0x000000000047d957 in tcache_alloc_small (slow_path=, zero=false, binind=3, size=, tcache=0x7ff9507e3010, arena=0x7ff961a01040, tsd=) at include/jemalloc/internal/tsd.h:283 #9 arena_malloc (slow_path=, tcache=0x7ff9507e3010, zero=false, ind=3, size=, arena=0x0, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:151 #10 iallocztm (slow_path=, arena=0x0, is_internal=false, tcache=0x7ff9507e3010, zero=false, ind=3, size=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:55 #11 imalloc_no_sample (ind=3, usize=48, size=, tsd=, dopts=, sopts=) at src/jemalloc.c:2398 #12 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2573 #13 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #14 je_malloc_default (size=) at src/jemalloc.c:2722 #15 0x00007ffeb65db06c in get_super_group_ids () from /opt/tsg/framework/lib/libmaat4.so.4 #16 0x00007ffeb65de01f in group2group_runtime_get_super_groups () from /opt/tsg/framework/lib/libmaat4.so.4 #17 0x00007ffeb65d935a in maat_compile_state_update () from /opt/tsg/framework/lib/libmaat4.so.4 #18 0x00007ffeb65df523 in ip_runtime_scan () from /opt/tsg/framework/lib/libmaat4.so.4 #19 0x00007ffeb65c4548 in maat_scan_ipv4 () from /opt/tsg/framework/lib/libmaat4.so.4 #20 0x00007ffdf51649c4 in ?? () #21 0x0000000000000008 in ?? () #22 0x00007ff9507e00e8 in ?? () #23 0x00007ff42b75e120 in ?? () #24 0x00007ff9507e0110 in ?? () #25 0x00007ff9507e07c0 in ?? () #26 0x00007ffd00000000 in ?? () #27 0x00007ffdf53fc860 in ?? () #28 0x0000000000000000 in ?? () Thread 9 (Thread 0x7ff94ffe2700 (LWP 4091)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007ffb4280a356 in ?? () #4 0x0000000000000000 in ?? () Thread 8 (Thread 0x7ff94f7e1700 (LWP 4092)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff960203a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff960203a28) at src/mutex.c:90 #5 0x00000000004dc830 in malloc_mutex_lock (mutex=0x7ff960203a28, tsdn=0x7ff94f7e0cb8) at include/jemalloc/internal/mutex.h:217 #6 je_extent_record (tsdn=tsdn@entry=0x7ff94f7e0cb8, pac=pac@entry=0x7ff9602039f0, ehooks=0x7ff9602000c0, ecache=ecache@entry=0x7ff960203a28, edata=edata@entry=0x7ff572a96280) at src/extent.c:921 #7 0x00000000004dc8e4 in je_ecache_dalloc (tsdn=tsdn@entry=0x7ff94f7e0cb8, pac=pac@entry=0x7ff9602039f0, ehooks=, ecache=ecache@entry=0x7ff960203a28, edata=edata@entry=0x7ff572a96280) at src/extent.c:147 #8 0x00000000004e8638 in pac_dalloc_impl (tsdn=0x7ff94f7e0cb8, self=0x7ff9602039f0, edata=0x7ff572a96280, deferred_work_generated=0x7ff94f7dfaef) at src/pac.c:277 #9 0x000000000048c6c2 in je_arena_slab_dalloc (tsdn=tsdn@entry=0x7ff94f7e0cb8, arena=0x7ff960201040, slab=) at src/arena.c:570 #10 0x00000000004ff55a in tcache_bin_flush_impl (small=true, nflush=0, ptrs=0x7ff94f7dfbc0, binind=, cache_bin=, tcache=, tsd=) at include/jemalloc/internal/atomic.h:83 #11 tcache_bin_flush_bottom (small=true, rem=, binind=, cache_bin=, tcache=, tsd=) at src/tcache.c:519 #12 je_tcache_bin_flush_small (tsd=, tcache=, cache_bin=, binind=, rem=) at src/tcache.c:529 #13 0x0000000000500326 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:430 #14 0x0000000000502c49 in je_te_event_trigger (tsd=0x7ff94f7e0cb8, ctx=) at src/thread_event.c:299 #15 0x0000000000481a60 in te_event_advance (is_alloc=, usize=, tsd=) at include/jemalloc/internal/thread_event.h:287 #16 thread_dalloc_event (usize=, tsd=) at include/jemalloc/internal/thread_event.h:293 #17 ifree (slow_path=, tcache=, ptr=, tsd=) at src/jemalloc.c:2896 #18 je_free_default (ptr=) at src/jemalloc.c:3021 #19 0x00007ffaafc008a0 in ?? () #20 0x00007fff0681d1bc in ?? () #21 0x00007ffaafc04913 in ?? () #22 0x0000000000000001 in ?? () #23 0x0000000000000025 in ?? () #24 0x0000000000000002 in ?? () #25 0x00007ffaafe199cf in ?? () #26 0x0000000000000025 in ?? () #27 0x0000000000000002 in ?? () #28 0x0000000000000000 in ?? () Thread 7 (Thread 0x7ff94efe0700 (LWP 4093)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff95ea03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff95ea03a28) at src/mutex.c:90 #5 0x00000000004dc830 in malloc_mutex_lock (mutex=0x7ff95ea03a28, tsdn=0x7ff94efdfcb8) at include/jemalloc/internal/mutex.h:217 #6 je_extent_record (tsdn=tsdn@entry=0x7ff94efdfcb8, pac=pac@entry=0x7ff95ea039f0, ehooks=0x7ff95ea000c0, ecache=ecache@entry=0x7ff95ea03a28, edata=edata@entry=0x7ff1a7866580) at src/extent.c:921 #7 0x00000000004dc8e4 in je_ecache_dalloc (tsdn=tsdn@entry=0x7ff94efdfcb8, pac=pac@entry=0x7ff95ea039f0, ehooks=, ecache=ecache@entry=0x7ff95ea03a28, edata=edata@entry=0x7ff1a7866580) at src/extent.c:147 #8 0x00000000004e8638 in pac_dalloc_impl (tsdn=0x7ff94efdfcb8, self=0x7ff95ea039f0, edata=0x7ff1a7866580, deferred_work_generated=0x7ff94efdefaf) at src/pac.c:277 #9 0x00000000004e5dd0 in large_dalloc_finish_impl (edata=0x7ff1a7866580, arena=0x7ff95ea01040, tsdn=0x7ff94efdfcb8) at src/large.c:253 #10 je_large_dalloc (tsdn=tsdn@entry=0x7ff94efdfcb8, edata=0x7ff1a7866580) at src/large.c:273 #11 0x0000000000481b63 in arena_dalloc_large (slow_path=false, szind=, tcache=, ptr=, tsdn=0x7ff94efdfcb8) at include/jemalloc/internal/arena_inlines_b.h:297 #12 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:334 #13 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #14 ifree (slow_path=false, tcache=0x7ff94efe0010, ptr=, tsd=0x7ff94efdfcb8) at src/jemalloc.c:2887 #15 je_free_default (ptr=) at src/jemalloc.c:3014 #16 0x00007ffdf516711f in ?? () #17 0x0000000000000000 in ?? () Thread 6 (Thread 0x7ff94e7df700 (LWP 4099)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffff3814d08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffff3814d08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffff3814d08, tsdn=0x13c48) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=100, ptrs=0x7ff94e7ddea0, binind=, cache_bin=, tcache=, tsd=0x13c48) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=4085337352, cache_bin=, tcache=, tsd=tsd@entry=0x13c48) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff94e7decb8, tcache=, cache_bin=, binind=binind@entry=9, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=9, ptr=0x7ffbd79e9760, tcache=, tsd=0x7ff94e7decb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff94e7df010, ptr=0x7ffbd79e9760, tsd=0x7ff94e7decb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ffbd79e9760) at src/jemalloc.c:3014 #14 0x0000000000482515 in free (ptr=) at src/jemalloc.c:3162 #15 0x0000000000458b54 in sapp_mem_free (type=type@entry=SAPP_MEM_DYN_PLUG_CTRL, thread_seq=, data=data@entry=0x7ffbd79e9764) at /home/yangwei/platform/sapp/src/common/sapp_mem.c:137 #16 0x000000000043630a in stream_process (a_stream=a_stream@entry=0x7fff109e70fc, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, funnum=19, pfunAarry=pfunAarry@entry=0x9946c0 , apme=0x7ffc1200c494, opstate=0x7fff109e7118 "\002\002", smart_offload_flag=1 '\001') at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1022 #17 0x00000000004368a7 in stream_process_udp (a_udp=a_udp@entry=0x7fff109e70fc, this_iphdr=this_iphdr@entry=0x0, transport_hdr=transport_hdr@entry=0x0, raw_pkt=raw_pkt@entry=0x0, apme=apme@entry=0x7ffc1200c494, popstate=popstate@entry=0x7fff109e7118 "\002\002") at /home/yangwei/platform/sapp/src/dealpkt/callapp.c:1125 #18 0x00000000004444cd in udp_free_stream (pindex=0x7fff109e70a4) at /home/yangwei/platform/sapp/src/dealpkt/deal_udp.c:323 #19 0x0000000000446862 in del_stream_by_time (plist=, current_drive_index=current_drive_index@entry=0x0, thread_id=thread_id@entry=39) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:523 #20 0x0000000000449a40 in polling_stream_timeout (tid=tid@entry=39) at /home/yangwei/platform/sapp/src/dealpkt/stream_manage.c:2720 #21 0x0000000000508bdd in marsio4_worker (arg=) at /home/yangwei/platform/sapp/src/packet_io/packet_io_marsio.c:1016 #22 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #23 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7ff94dfde700 (LWP 4102)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffebd015e08) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffebd015e08) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffebd015e08, tsdn=0x14dc8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff94dfdcc50, binind=, cache_bin=, tcache=, tsd=0x14dc8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3170983432, cache_bin=, tcache=, tsd=tsd@entry=0x14dc8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff94dfddcb8, tcache=, cache_bin=, binind=binind@entry=29, rem=) at src/tcache.c:529 #9 0x0000000000481af8 in tcache_dalloc_small (slow_path=false, binind=29, ptr=0x7ff16625d800, tcache=, tsd=0x7ff94dfddcb8) at include/jemalloc/internal/tcache_inlines.h:157 #10 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:331 #11 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #12 ifree (slow_path=false, tcache=0x7ff94dfde010, ptr=0x7ff16625d800, tsd=0x7ff94dfddcb8) at src/jemalloc.c:2887 #13 je_free_default (ptr=0x7ff16625d800) at src/jemalloc.c:3014 #14 0x00007ffaafc008a0 in ?? () #15 0x00007fff13a8253c in ?? () #16 0x00007ffaafc04913 in ?? () #17 0x0000000000000001 in ?? () #18 0x0000000000000028 in ?? () #19 0x0000000000000002 in ?? () #20 0x00007ffaafe199cf in ?? () #21 0x0000000000000028 in ?? () #22 0x0000000000000002 in ?? () #23 0x0000000000000000 in ?? () Thread 4 (Thread 0x7ff94d7dd700 (LWP 4103)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffebc214f28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffebc214f28) at src/mutex.c:90 #5 0x00000000004ff698 in malloc_mutex_lock (mutex=0x7ffebc214f28, tsdn=0x13ee8) at include/jemalloc/internal/mutex.h:217 #6 tcache_bin_flush_impl (small=true, nflush=10, ptrs=0x7ff94d7dbbe0, binind=, cache_bin=, tcache=, tsd=0x13ee8) at src/tcache.c:376 #7 tcache_bin_flush_bottom (small=true, rem=, binind=3156299560, cache_bin=0x7ff94d7dbc20, tcache=0x2, tsd=tsd@entry=0x13ee8) at src/tcache.c:519 #8 je_tcache_bin_flush_small (tsd=tsd@entry=0x7ff94d7dccb8, tcache=tcache@entry=0x7ff94d7dd010, cache_bin=cache_bin@entry=0x7ff94d7dd138, binind=binind@entry=12, rem=) at src/tcache.c:529 #9 0x00000000005004e6 in tcache_gc_small (szind=12, tcache=0x7ff94d7dd010, tcache_slow=0x7ff94d7dcdb8, tsd=0x7ff94d7dccb8) at src/tcache.c:148 #10 tcache_event (tsd=tsd@entry=0x7ff94d7dccb8) at src/tcache.c:193 #11 je_tcache_gc_event_handler (tsd=tsd@entry=0x7ff94d7dccb8, elapsed=elapsed@entry=18446744073709551615) at src/tcache.c:219 #12 0x0000000000502d9b in je_te_event_trigger (tsd=0x7ff94d7dccb8, ctx=) at src/thread_event.c:299 #13 0x000000000047d4b1 in te_event_advance (is_alloc=true, usize=5120, tsd=) at include/jemalloc/internal/thread_event.h:287 #14 thread_alloc_event (usize=5120, tsd=) at include/jemalloc/internal/thread_event.h:298 #15 imalloc_body (tsd=, dopts=, sopts=) at src/jemalloc.c:2585 #16 imalloc (dopts=, sopts=) at src/jemalloc.c:2687 #17 je_malloc_default (size=) at src/jemalloc.c:2722 #18 0x00007ffdf5168efa in ?? () #19 0x0000000000000000 in ?? () Thread 3 (Thread 0x7ff94cfdc700 (LWP 4104)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ffeb4e03a28) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ffeb4e03a28) at src/mutex.c:90 #5 0x00000000004dc830 in malloc_mutex_lock (mutex=0x7ffeb4e03a28, tsdn=0x7ff94cfdbcb8) at include/jemalloc/internal/mutex.h:217 #6 je_extent_record (tsdn=tsdn@entry=0x7ff94cfdbcb8, pac=pac@entry=0x7ffeb4e039f0, ehooks=0x7ffeb4e000c0, ecache=ecache@entry=0x7ffeb4e03a28, edata=edata@entry=0x7ff1a6e6f480) at src/extent.c:921 #7 0x00000000004dc8e4 in je_ecache_dalloc (tsdn=tsdn@entry=0x7ff94cfdbcb8, pac=pac@entry=0x7ffeb4e039f0, ehooks=, ecache=ecache@entry=0x7ffeb4e03a28, edata=edata@entry=0x7ff1a6e6f480) at src/extent.c:147 #8 0x00000000004e8638 in pac_dalloc_impl (tsdn=0x7ff94cfdbcb8, self=0x7ffeb4e039f0, edata=0x7ff1a6e6f480, deferred_work_generated=0x7ff94cfdafaf) at src/pac.c:277 #9 0x00000000004e5dd0 in large_dalloc_finish_impl (edata=0x7ff1a6e6f480, arena=0x7ffeb4e01040, tsdn=0x7ff94cfdbcb8) at src/large.c:253 #10 je_large_dalloc (tsdn=tsdn@entry=0x7ff94cfdbcb8, edata=0x7ff1a6e6f480) at src/large.c:273 #11 0x0000000000481b63 in arena_dalloc_large (slow_path=false, szind=, tcache=, ptr=, tsdn=0x7ff94cfdbcb8) at include/jemalloc/internal/arena_inlines_b.h:297 #12 arena_dalloc (slow_path=, caller_alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/arena_inlines_b.h:334 #13 idalloctm (slow_path=, is_internal=, alloc_ctx=, tcache=, ptr=, tsdn=) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:120 #14 ifree (slow_path=false, tcache=0x7ff94cfdc010, ptr=, tsd=0x7ff94cfdbcb8) at src/jemalloc.c:2887 #15 je_free_default (ptr=) at src/jemalloc.c:3014 #16 0x00007ffdf516711f in ?? () #17 0x0000000000000000 in ?? () Thread 2 (Thread 0x7ff94c7db700 (LWP 4107)): #0 0x00007ffff59999fd in nanosleep () from /lib64/libc.so.6 #1 0x00007ffff59ca2d4 in usleep () from /lib64/libc.so.6 #2 0x0000000000469a1a in sapp_monitor_thread (args=) at /home/yangwei/platform/sapp/src/sapp_dev/sapp_plug.c:219 #3 0x00007ffff690eea5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007ffff59d2b0d in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7ffff7fd6300 (LWP 3923)): #0 0x00007ffff691554d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007ffff6910e9b in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007ffff6910d68 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00000000004e7075 in malloc_mutex_lock_final (mutex=0x7ff961a14748) at include/jemalloc/internal/mutex.h:151 #4 je_malloc_mutex_lock_slow (mutex=mutex@entry=0x7ff961a14748) at src/mutex.c:90 #5 0x00000000004e7250 in malloc_mutex_lock (mutex=mutex@entry=0x7ff961a14748, tsdn=tsdn@entry=0x7ffff7fd58b8) at include/jemalloc/internal/mutex.h:217 #6 je_malloc_mutex_prefork (tsdn=tsdn@entry=0x7ffff7fd58b8, mutex=mutex@entry=0x7ff961a14748) at src/mutex.c:191 #7 0x0000000000496315 in je_bin_prefork (tsdn=tsdn@entry=0x7ffff7fd58b8, bin=bin@entry=0x7ff961a14748) at src/bin.c:58 #8 0x0000000000491b14 in je_arena_prefork8 (tsdn=0x7ffff7fd58b8, arena=0x7ff961a01040) at src/arena.c:1837 #9 0x0000000000477295 in je_jemalloc_prefork () at include/jemalloc/internal/tsd.h:283 #10 0x00007ffff5999a7c in fork () from /lib64/libc.so.6 #11 0x00007ffff6d566c9 in breakpad_dump_and_report(google_breakpad::MinidumpDescriptor const&, void*, bool) () from /opt/tsg/framework/lib/libbreakpad_mini.so #12 0x00007ffff6d57a74 in google_breakpad::ExceptionHandler::GenerateDump(google_breakpad::ExceptionHandler::CrashContext*) () from /opt/tsg/framework/lib/libbreakpad_mini.so #13 0x00007ffff6d57da7 in google_breakpad::ExceptionHandler::SignalHandler(int, siginfo_t*, void*) () from /opt/tsg/framework/lib/libbreakpad_mini.so #14 #15 0x00007ffff6915f5d in pause () from /lib64/libpthread.so.0 #16 0x0000000000421975 in main (argc=, argv=) at /home/yangwei/platform/sapp/src/entry/sapp_main.c:28