$ pstack 7916
Thread 17 (Thread 1107855712 (LWP 8262)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 16 (Thread 1118345568 (LWP 8263)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 15 (Thread 1128835424 (LWP 8264)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 14 (Thread 1139325280 (LWP 8265)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 13 (Thread 1149815136 (LWP 8266)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 12 (Thread 1160304992 (LWP 8267)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 11 (Thread 1170794848 (LWP 8268)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 10 (Thread 1085798752 (LWP 8270)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 9 (Thread 1096288608 (LWP 8271)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 8 (Thread 1181284704 (LWP 8273)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 7 (Thread 1191774560 (LWP 8276)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 6 (Thread 1202264416 (LWP 8277)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 5 (Thread 1212754272 (LWP 8278)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 4 (Thread 1223244128 (LWP 8279)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 3 (Thread 1233733984 (LWP 8280)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 2 (Thread 1244223840 (LWP 8281)):
#0 0x0000003f0b90a65f in sem_wait () from /lib64/tls/libpthread.so.0
#1 0x00007f0d88cdbeb8 in PyThread_acquire_lock ()
#2 0x00007f0d88cdf602 in lock_PyThread_acquire_lock ()
#3 0x00007f0d88cb19cb in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#5 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#7 0x00007f0d88cb1eee in PyEval_EvalFrameEx ()
#8 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#9 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#10 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#11 0x00007f0d88c457e8 in function_call ()
#12 0x00007f0d88c2189d in PyObject_Call ()
#13 0x00007f0d88c30c14 in instancemethod_call ()
#14 0x00007f0d88c2189d in PyObject_Call ()
#15 0x00007f0d88cac3e1 in PyEval_CallObjectWithKeywords ()
#16 0x00007f0d88cdff81 in t_bootstrap ()
#17 0x0000003f0b90610a in start_thread () from /lib64/tls/libpthread.so.0
#18 0x0000003f0b0c5ee3 in clone () from /lib64/tls/libc.so.6
#19 0x0000000000000000 in ?? ()
Thread 1 (Thread 139696105322208 (LWP 7916)):================================》主线程
#0 0x0000003f0b0bd0c2 in poll () from /lib64/tls/libc.so.6
#1 0x00007f0d853c5730 in internal_select ()
#2 0x00007f0d853c61ca in sock_accept ()
#3 0x00007f0d88cafa21 in PyEval_EvalFrameEx ()
#4 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#5 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#6 0x00007f0d88cb2ce6 in PyEval_EvalFrameEx ()
#7 0x00007f0d88cb3624 in PyEval_EvalCodeEx ()
#8 0x00007f0d88cb3822 in PyEval_EvalCode ()
#9 0x00007f0d88ccc49c in run_mod ()
#10 0x00007f0d88cccafc in PyRun_FileExFlags ()
#11 0x00007f0d88ccda11 in PyRun_SimpleFileExFlags ()
#12 0x00007f0d88cddd26 in Py_Main ()
#13 0x0000003f0b01c4bb in __libc_start_main () from /lib64/tls/libc.so.6
#14 0x00000000004006da in _start ()
#15 0x00007fff62a773d8 in ?? ()
#16 0x000000000000001c in ?? ()
#17 0x0000000000000003 in ?? ()
#18 0x00007fff62a776db in ?? ()
#19 0x00007fff62a776e5 in ?? ()
#20 0x00007fff62a7771e in ?? ()
#21 0x0000000000000000 in ?? ()
----------------------------------------------------------------
dump_time=2021-04-09 12:19:55 reqid=2A5BE75641F1BC5A
1 ActiveThreadCount: 18
2 KnownActiveThreadNames:
MainThread-------------------------------- 主线程
APScheduler------------------------------- 任务调度线程
CP Server Thread-1------------------------Butterfly handler 处理线程,以下类似
CP Server Thread-11
CP Server Thread-5
CP Server Thread-9
CP Server Thread-14
CP Server Thread-7
CP Server Thread-13
CP Server Thread-15
CP Server Thread-3
CP Server Thread-2
CP Server Thread-4
CP Server Thread-6
CP Server Thread-12
CP Server Thread-8
CP Server Thread-10
CP Server Thread-16
3 ThreadInfo:
Thread: 1082784096 CP Server Thread-16
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1093196128 CP Server Thread-15
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1089005920 CP Server Thread-3
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1076640096 CP Server Thread-13
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1103497568 CP Server Thread-6
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1094404448 CP Server Thread-5
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1100007776 CP Server Thread-2
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 140004998178528 MainThread
File: "/home/users/meetbill/butterfly/main.py", line 30, in <module>
server.start()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 2104, in start
self.tick()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 2158, in tick
s, addr = self.socket.accept()
File: "/home/users/meetbill/.jumbo/lib/python2.7/socket.py", line 202, in accept
sock, addr = self._sock.accept()
Thread: 1102469472 CP Server Thread-7
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1095801184 CP Server Thread-8
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1100343648 CP Server Thread-11
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1075272032 CP Server Thread-14
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1100863840 APScheduler
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 504, in run
self.__target(*self.__args, **self.__kwargs)
File: "/home/users/meetbill/butterfly/xlib/apscheduler/schedulers/blocking.py", line 42, in _main_loop
self._event.wait(wait_seconds)
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 403, in wait
self.__cond.wait(timeout)
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 262, in wait
_sleep(delay)
Thread: 1084520800 CP Server Thread-4
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1097337184 CP Server Thread-9
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1091410272 CP Server Thread-12
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1087252832 CP Server Thread-10
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1643, in run
conn = self.server.requests.get()
File: "/home/users/meetbill/.jumbo/lib/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 243, in wait
waiter.acquire()
Thread: 1099741536 CP Server Thread-1
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 524, in __bootstrap
self.__bootstrap_inner()
File: "/home/users/meetbill/.jumbo/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1651, in run
conn.communicate()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 1460, in communicate
req.respond()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 901, in respond
self.server.gateway(self).respond()
File: "/home/users/meetbill/butterfly/xlib/cherrypy_wsgiserver/__init__.py", line 2426, in respond
response = self.req.server.wsgi_app(self.env, self.start_response)
File: "/home/users/meetbill/butterfly/wsgiapp.py", line 161, in application
status, headders, content = wsgigw.process(environ)
File: "/home/users/meetbill/butterfly/xlib/httpgateway.py", line 195, in process
httpstatus, headers, content = protocol(req)
File: "/home/users/meetbill/butterfly/xlib/protocol_json.py", line 171, in __call__
ret = self._func(**params)
File: "/home/users/meetbill/butterfly/handlers/demo_stackdump/__init__.py", line 79, in stackdump
for filename, lineno, name, line in traceback.extract_stack(stack):
# gdb `which python ` core.python2.7.25133.1617075053
...
gdb info
...
[New LWP 25192]
[New LWP 25193]
[New LWP 25195]
[New LWP 25196]
[New LWP 25197]
[New LWP 25194]
[New LWP 25198]
[New LWP 25199]
[New LWP 25201]
[New LWP 25200]
[New LWP 25202]
[New LWP 25204]
[New LWP 25206]
[New LWP 25203]
[New LWP 25205]
[New LWP 25207]
[New LWP 25191]
[New LWP 25208]
[New LWP 25133]
warning: File "/lib64/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".
To enable execution of this file add
add-auto-load-safe-path /lib64/libthread_db-1.0.so
line to your configuration file "/home/work/.gdbinit".
To completely disable this security protection add
set auto-load safe-path /
line to your configuration file "/home/work/.gdbinit".
For more information about this security protection see the
"Auto-loading safe path" section in the GDB manual. E.g., run from the shell:
info "(gdb)Auto-loading safe path"
warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
warning: File "/lib64/libthread_db-1.0.so" auto-loading has been declined by your `auto-load safe-path' set to "$debugdir:$datadir/auto-load".
warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
Core was generated by `python2.7 /home/work/chunfeng/chunfeng_app2_4600/main.py 954216e9'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 0x00007f85e85265c5 in PyEval_EvalFrameEx (f=<optimized out>, throwflag=<optimized out>) at Python/ceval.c:2995
2995 Python/ceval.c: No such file or directory.
[Current thread is 1 (LWP 25192)]
(gdb)
#0 0x00007f85e85265c5 in PyEval_EvalFrameEx (f=<optimized out>, throwflag=<optimized out>) at Python/ceval.c:2995