I have been experiencing random crash with opnsense and it would be helpful if someone could help with me debugging the cause of this crash.
Trying to debug with chatgpt, it suggested turning off ICMP redirects. have set net.inet.icmp.drop_redirect to 1 but didn't seem to have made a difference.
db:0:kdb.enter.default> alltrace
Tracing command kernel pid 0 tid 100000 td 0xffffffff81b80d20
sched_switch() at sched_switch+0x88a/frame 0xffffffff8383eec0
mi_switch() at mi_switch+0xbb/frame 0xffffffff8383eee0
sleepq_timedwait() at sleepq_timedwait+0x2f/frame 0xffffffff8383ef20
_sleep() at _sleep+0x1c8/frame 0xffffffff8383efa0
swapper() at swapper+0x69/frame 0xffffffff8383eff0
Tracing command kernel pid 0 tid 100007 td 0xfffff800047e7740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149aee50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149aee70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149aeec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149aeef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149aef30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149aef30
--- trap 0x91828944, rip = 0x4d167c9a1f82d9b, rsp = 0x85ff20d66a84fe21, rbp = 0x5d236caf0fb7c986 ---
Tracing command kernel pid 0 tid 100008 td 0xfffff800047e7000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149a9e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149a9e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149a9ec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149a9ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149a9f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149a9f30
--- trap 0x48578d48, rip = 0x44f8d483275c385, rsp = 0x483c458d45084f8d, rbp = 0x211840fce2b4100 ---
Tracing command kernel pid 0 tid 100009 td 0xfffff800047c7740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149f4e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149f4e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149f4ec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149f4ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149f4f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149f4f30
--- trap 0xa070707, rip = 0xa19017006320a, rsp = 0x71101c0153219, rbp = 0x3200000001f ---
Tracing command kernel pid 0 tid 100010 td 0xfffff800047c7000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149efe50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149efe70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149efec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149efef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149eff30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149eff30
--- trap 0x2b4904eb, rip = 0x8b49028a4142518d, rsp = 0x41894938418d4908, rbp = 0xccccccc35f20c483 ---
Tracing command kernel pid 0 tid 100011 td 0xfffff800047c8740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149eae50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149eae70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149eaec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149eaef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149eaf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149eaf30
--- trap 0x22808a41, rip = 0x410000015a828800, rsp = 0x1528288000003, rbp = 0x20808a410000014d ---
Tracing command kernel pid 0 tid 100012 td 0xfffff800047c8000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149e5e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149e5e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149e5ec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149e5ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149e5f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149e5f30
--- trap 0x48000000, rip = 0x8d44000000b8249c, rsp = 0x99850fc58548, rbp = 0x666824448d486850 ---
Tracing command kernel pid 0 tid 100013 td 0xfffff800047c9740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149e0e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149e0e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149e0ec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149e0ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149e0f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149e0f30
--- trap 0x4024448a, rip = 0x440a28245c894858, rsp = 0x24448820245c8948, rbp = 0x448d4c40244c8d4c ---
Tracing command kernel pid 0 tid 100014 td 0xfffff800047c9000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149dbe50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149dbe70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149dbec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149dbef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149dbf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149dbf30
--- trap 0x314e1c1, rip = 0xe2c100000cf8b9d1, rsp = 0xb000000fc2508e2, rbp = 0xf106e800000030b8 ---
Tracing command kernel pid 0 tid 100015 td 0xfffff800047ca740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149d6e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149d6e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00149d6ec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xf1/frame 0xfffffe00149d6ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149d6f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149d6f30
--- trap 0x4d363603, rip = 0xd3d3d0000000000, rsp = 0x2524233322121420, rbp = 0x500000024b000004 ---
Tracing command kernel pid 0 tid 100016 td 0xfffff800047ca000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149d1e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149d1e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00149d1ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00149d1ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149d1f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149d1f30
--- trap 0x775004c, rip = 0x4a788041077309e0, rsp = 0xbfcf2244cd2244cc, rbp = 0xa801b10274004978 ---
Tracing command kernel pid 0 tid 100017 td 0xfffff800047bf740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a34f5e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a34f5e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a34f5ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a34f5ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a34f5f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a34f5f30
--- trap 0x8147cb15, rip = 0x154281e7a8adfa39, rsp = 0x2efed9433638e8c, rbp = 0xd2977efc463a77b6 ---
Tracing command kernel pid 0 tid 100020 td 0xfffff800047c0000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a34e6e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a34e6e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a34e6ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a34e6ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a34e6f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a34e6f30
--- trap 0x915a915a, rip = 0x349eea6957860c58, rsp = 0x8542092111e7206b, rbp = 0x979e8f58bed4ddcc ---
Tracing command kernel pid 0 tid 100022 td 0xfffff800047c1000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a34dce20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a34dce40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a34dcec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a34dcef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a34dcf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a34dcf30
--- trap 0x6a07ac36, rip = 0xba5fa2999315f00d, rsp = 0x3535da4e04b9b956, rbp = 0xab35b2f3827fe167 ---
Tracing command kernel pid 0 tid 100023 td 0xfffff800047c2740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149fee20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149fee40
_sleep() at _sleep+0x1e8/frame 0xfffffe00149feec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00149feef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149fef30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149fef30
--- trap 0x8b48c933, rip = 0x4800000d89158b48, rsp = 0x85158d4800000d84, rbp = 0x4818498b48402444 ---
Tracing command kernel pid 0 tid 100024 td 0xfffff800047c2000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00149f9e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00149f9e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00149f9ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00149f9ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149f9f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149f9f30
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
Tracing command kernel pid 0 tid 100025 td 0xfffff800047b7740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a351de20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a351de40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a351dec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a351def0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a351df30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a351df30
--- trap 0xfe3585eb, rip = 0x9148f0503661ba02, rsp = 0xa257e8057ca055ea, rbp = 0x51f11ba38f06a64c ---
Tracing command kernel pid 0 tid 100026 td 0xfffff800047b7000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3518e50
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3518e70
msleep_spin_sbt() at msleep_spin_sbt+0x186/frame 0xfffffe00a3518ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xf1/frame 0xfffffe00a3518ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3518f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3518f30
--- trap 0xe12f8217, rip = 0x5ac89ce9109a085, rsp = 0xb45e20f309b15b25, rbp = 0x31398cd6f70800b5 ---
Tracing command kernel pid 0 tid 100027 td 0xfffff800047bc740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3513e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3513e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3513ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3513ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3513f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3513f30
--- trap 0x5b9e3886, rip = 0xc3b9a0216690e8f3, rsp = 0x7c6fc180ba5e4de3, rbp = 0xc802880288228802 ---
Tracing command kernel pid 0 tid 100028 td 0xfffff800047bc000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a350ee20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a350ee40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a350eec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a350eef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a350ef30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a350ef30
--- trap 0x8cadf773, rip = 0x5adae7359ceb6b56, rsp = 0x95b16bcaa43d19d2, rbp = 0xbe60fd781349972a ---
Tracing command kernel pid 0 tid 100029 td 0xfffff800047bd740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3509e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3509e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3509ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3509ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3509f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3509f30
--- trap 0xffa58c18, rip = 0x824b7db0a3471ea8, rsp = 0x1b05324f60dbc5f2, rbp = 0x410c410c418c610c ---
Tracing command kernel pid 0 tid 100030 td 0xfffff800047bd000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3504e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3504e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3504ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3504ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3504f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3504f30
--- trap 0x6a17ac36, rip = 0xba5fa2999315f00d, rsp = 0x3535da4e04bdb95e, rbp = 0xab35b2f3827fe167 ---
Tracing command kernel pid 0 tid 100031 td 0xfffff800047be740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a34ffe20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a34ffe40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a34ffec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a34ffef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a34fff30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a34fff30
--- trap 0x903f049a, rip = 0x2031fe86432938f7, rsp = 0xe112920a543bd858, rbp = 0xd131aaef5dd2b629 ---
Tracing command kernel pid 0 tid 100032 td 0xfffff800047be000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a34fae20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a34fae40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a34faec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a34faef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a34faf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a34faf30
--- trap 0x9aa38265, rip = 0xc1181feaa200d9de, rsp = 0xc43b020e8e69968d, rbp = 0x672eeb4cf38ac206 ---
Tracing command kernel pid 0 tid 100033 td 0xfffff800047b3740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3545e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3545e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3545ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3545ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3545f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3545f30
--- trap 0xf1427d21, rip = 0xcca29e363b1f714d, rsp = 0xdb9ddb9ddb9dfb9d, rbp = 0xd9a9d9a9d9a9d9a9 ---
Tracing command kernel pid 0 tid 100034 td 0xfffff800047b3000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3540e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3540e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3540ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3540ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3540f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3540f30
--- trap 0x69e1d40e, rip = 0x21b8397e08f26bea, rsp = 0x1314fc6732909f77, rbp = 0xa9bd80f7d263774a ---
Tracing command kernel pid 0 tid 100035 td 0xfffff800047b4740
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a353be20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a353be40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a353bec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a353bef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a353bf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a353bf30
--- trap 0x73465a8c, rip = 0xacb48995c3c75762, rsp = 0x7b0c00d0f76d181e, rbp = 0x2221d59cbae7e410 ---
Tracing command kernel pid 0 tid 100036 td 0xfffff800047b4000
sched_switch() at sched_switch+0x88a/frame 0xfffffe00a3536e20
mi_switch() at mi_switch+0xbb/frame 0xfffffe00a3536e40
_sleep() at _sleep+0x1e8/frame 0xfffffe00a3536ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xfffffe00a3536ef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00a3536f30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00a3536f30
--- trap 0xaaac1743, rip = 0x820f830f820f820f, rsp = 0xa15cda822d3fc244, rbp = 0xe042d1ceb2d476e7 ---
Tracing command kernel pid 0 tid 100045 td 0xfffff800047a1740
sched_switch() at sched_switch+0x88a/frame 0xffffffff82850e20
mi_switch() at mi_switch+0xbb/frame 0xffffffff82850e40
_sleep() at _sleep+0x1e8/frame 0xffffffff82850ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff82850ef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff82850f30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff82850f30
--- trap 0xa99db15b, rip = 0xc276dab0eb3c8824, rsp = 0x2930bd9794dfc64b, rbp = 0xb05fa8999915fa0d ---
Tracing command kernel pid 0 tid 100050 td 0xfffff80004a47000
sched_switch() at sched_switch+0x88a/frame 0xffffffff82897e20
mi_switch() at mi_switch+0xbb/frame 0xffffffff82897e40
_sleep() at _sleep+0x1e8/frame 0xffffffff82897ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff82897ef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff82897f30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff82897f30
--- trap 0x21881004, rip = 0xb0f85f8381743c9b, rsp = 0x341bc3a62cdd722a, rbp = 0x9e7baff38cef0ade ---
Tracing command kernel pid 0 tid 100051 td 0xfffff80004a48740
sched_switch() at sched_switch+0x88a/frame 0xffffffff82892e20
mi_switch() at mi_switch+0xbb/frame 0xffffffff82892e40
_sleep() at _sleep+0x1e8/frame 0xffffffff82892ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff82892ef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff82892f30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff82892f30
--- trap 0x6c6cba1b, rip = 0xaed83a7d1b3740e3, rsp = 0x866a124f3b856811, rbp = 0xde26d726c626d626 ---
Tracing command kernel pid 0 tid 100052 td 0xfffff80004a48000
sched_switch() at sched_switch+0x88a/frame 0xffffffff8288de20
mi_switch() at mi_switch+0xbb/frame 0xffffffff8288de40
_sleep() at _sleep+0x1e8/frame 0xffffffff8288dec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff8288def0
fork_exit() at fork_exit+0x7f/frame 0xffffffff8288df30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff8288df30
--- trap 0x53dbbca0, rip = 0x197c013a3036532e, rsp = 0x58aa1af00655af1f, rbp = 0xe72f5ac82116d6a ---
Tracing command kernel pid 0 tid 100053 td 0xfffff80004a49740
sched_switch() at sched_switch+0x88a/frame 0xffffffff82888e20
mi_switch() at mi_switch+0xbb/frame 0xffffffff82888e40
_sleep() at _sleep+0x1e8/frame 0xffffffff82888ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff82888ef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff82888f30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff82888f30
--- trap 0x9e7069cd, rip = 0x718e4403311ae52b, rsp = 0xc87e6d572705b3a0, rbp = 0xc1e9ae927065cd8a ---
Tracing command kernel pid 0 tid 100083 td 0xfffff8000501d740
sched_switch() at sched_switch+0x88a/frame 0xffffffff83413e20
mi_switch() at mi_switch+0xbb/frame 0xffffffff83413e40
_sleep() at _sleep+0x1e8/frame 0xffffffff83413ec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff83413ef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff83413f30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff83413f30
--- trap 0, rip = 0, rsp = 0, rbp = 0 ---
Tracing command kernel pid 0 tid 100084 td 0xfffff8000501d000
sched_switch() at sched_switch+0x88a/frame 0xffffffff8340ee20
mi_switch() at mi_switch+0xbb/frame 0xffffffff8340ee40
_sleep() at _sleep+0x1e8/frame 0xffffffff8340eec0
taskqueue_thread_loop() at taskqueue_thread_loop+0xb1/frame 0xffffffff8340eef0
fork_exit() at fork_exit+0x7f/frame 0xffffffff8340ef30
fork_trampoline() at fork_trampoline+0xe/frame 0xffffffff8340ef30
System information
FreeBSD 14.2-RELEASE-p2 stable/25.1-n269701-7c59d89f8cd SMP amd64
OPNsense 25.1.3 6aa1d97b1
Plugins os-acme-client-4.9 os-adguardhome-maxit-1.15 os-chrony-1.5_2 os-cpu-microcode-intel-1.1 os-ddclient-1.27 os-dmidecode-1.2 os-freeradius-1.9.27 os-lcdproc-sdeclcd-1.1_1 os-smart-2.3 os-telegraf-1.12.12 os-vnstat-1.3_1 os-wol-2.5_1
Time Mon, 24 Mar 2025 13:28:08 +0000
OpenSSL 3.0.16
Python 3.11.11
PHP 8.3.17
Processor: Intel(R) Core(TM) i3-4330 CPU @ 3.50GHz (3492.13-MHz K8-class CPU)
NIC: Intel(R) I210 (Copper)
Device: Sophos SG310
Do you have the actual backtrace too?
I think ChatGPT tells you something that somebody once suggested on the Internet regarding a bug that was happening at the time, but certainly not now. ;)
Cheers,
Franco
It crashed again so this is the backtrace from that. I guess this is the one
db:0:kdb.enter.default> bt
Tracing pid 0 tid 100011 td 0xfffff800047c8740
kdb_enter() at kdb_enter+0x33/frame 0xfffffe00149ea750
panic() at panic+0x43/frame 0xfffffe00149ea7b0
trap_fatal() at trap_fatal+0x40b/frame 0xfffffe00149ea810
trap_pfault() at trap_pfault+0x46/frame 0xfffffe00149ea860
calltrap() at calltrap+0x8/frame 0xfffffe00149ea860
--- trap 0xc, rip = 0xffffffff80d8c630, rsp = 0xfffffe00149ea938, rbp = 0xfffffe00149ea9f0 ---
ip_redir_alloc() at ip_redir_alloc/frame 0xfffffe00149ea9f0
ip_input() at ip_input+0x56c/frame 0xfffffe00149eaa50
netisr_dispatch_src() at netisr_dispatch_src+0x9e/frame 0xfffffe00149eaaa0
ether_demux() at ether_demux+0x149/frame 0xfffffe00149eaad0
ether_nh_input() at ether_nh_input+0x36a/frame 0xfffffe00149eab30
netisr_dispatch_src() at netisr_dispatch_src+0x9e/frame 0xfffffe00149eab80
ether_input() at ether_input+0x56/frame 0xfffffe00149eabd0
ether_demux() at ether_demux+0x8e/frame 0xfffffe00149eac00
ether_nh_input() at ether_nh_input+0x36a/frame 0xfffffe00149eac60
netisr_dispatch_src() at netisr_dispatch_src+0x9e/frame 0xfffffe00149eacb0
ether_input() at ether_input+0x56/frame 0xfffffe00149ead00
iflib_rxeof() at iflib_rxeof+0xc1e/frame 0xfffffe00149eae00
_task_fn_rx() at _task_fn_rx+0x72/frame 0xfffffe00149eae40
gtaskqueue_run_locked() at gtaskqueue_run_locked+0x14e/frame 0xfffffe00149eaec0
gtaskqueue_thread_loop() at gtaskqueue_thread_loop+0xc2/frame 0xfffffe00149eaef0
fork_exit() at fork_exit+0x7f/frame 0xfffffe00149eaf30
fork_trampoline() at fork_trampoline+0xe/frame 0xfffffe00149eaf30
--- trap 0xcccccccc, rip = 0x8a00d18009e0ba0f, rsp = 0x53552d6e6500, rbp = 0x36e81feaba0fd00b ---
I was able to narrow down the cause of the crash. I have openwrt router as next hop for some traffic and by default it rejects traffic instead of drop.
So whenever it rejects the traffic, it causes opnsense to crash. Changing it to drop stops it from crashing.
A pcap may be helpful here along with the OpenWRT details such as version, whether it's happening on the same OpenWRT version or multiple ones
And as a general rule, it would be best to have OpenWRT behind the FW and dumbed down to AP functionality only, with or without VLANs as the case may be.