After upgrade from 25.1.12 to 25.7.2: High response times when having traffic

[rainerle]

Hi,
after the upgrade on Tuesday the uplink delay went up from 1.2ms to 24ms.

Where can I look to fix this?

The measurement is taken from another machine behind the opnsense.

CPU usage between 2% and 12%,
More than 50% Memory free,
no idea where to look further...

Thanks
Rainer
 

[rainerle]

dmesg

Code Select Expand

root@opnsense01:~ # grep ixl2 /var/run/dmesg.boot
[1] ixl2: <Intel(R) Ethernet Connection X722 for 10GbE SFP+ - 2.3.3-k> mem 0x21ffb000000-0x21ffbffffff,0x21fff008000-0x21fff00ffff irq 36 at device 0.2 numa-domain 0 on pci6
[1] ixl2: fw 3.10.53077 api 1.5 nvm 4.00 etid 8000183c oem 1.263.0
[1] ixl2: PF-ID[2]: VFs 32, MSI-X 129, VF MSI-X 5, QPs 384, I2C
[1] ixl2: Using 1024 TX descriptors and 1024 RX descriptors
[1] ixl2: Using 8 RX queues 8 TX queues
[1] ixl2: Using MSI-X interrupts with 9 vectors
[1] ixl2: Ethernet address: 7c:d3:0a:d8:34:a2
[1] ixl2: Allocating 8 queues for PF LAN VSI; 8 queues active
[1] ixl2: SR-IOV ready
[1] ixl2: netmap queues/slots: TX 8/1024, RX 8/1024
[1] ixl2: Link is up, 10 Gbps Full Duplex, Requested FEC: None, Negotiated FEC: None, Autoneg: False, Flow Control: None
[1] ixl2: link state changed to UP
root@opnsense01:~ #

pciconf -lv

Code Select Expand

...
ixl0@pci0:10:0:0:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GbE SFP+'
    class      = network
    subclass   = ethernet
ixl1@pci0:10:0:1:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GbE SFP+'
    class      = network
    subclass   = ethernet
ixl2@pci0:10:0:2:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GbE SFP+'
    class      = network
    subclass   = ethernet
ixl3@pci0:10:0:3:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GbE SFP+'
    class      = network
    subclass   = ethernet
...


[rainerle]

Here the pciconf -l

Code Select Expand

root@opnsense01:~ # pciconf -l
hostb0@pci0:0:0:0:      class=0x060000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2020 subvendor=0x17aa subdevice=0x7800
ioat0@pci0:0:4:0:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat1@pci0:0:4:1:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat2@pci0:0:4:2:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat3@pci0:0:4:3:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat4@pci0:0:4:4:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat5@pci0:0:4:5:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat6@pci0:0:4:6:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
ioat7@pci0:0:4:7:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2021 subvendor=0x17aa subdevice=0x7800
none0@pci0:0:5:0:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2024 subvendor=0x17aa subdevice=0x7800
none1@pci0:0:5:2:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2025 subvendor=0x17aa subdevice=0x7800
ioapic0@pci0:0:5:4:     class=0x080020 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2026 subvendor=0x17aa subdevice=0x7800
none2@pci0:0:8:0:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2014 subvendor=0x17aa subdevice=0x7800
none3@pci0:0:8:1:       class=0x110100 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2015 subvendor=0x17aa subdevice=0x7800
none4@pci0:0:8:2:       class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2016 subvendor=0x17aa subdevice=0x7800
none5@pci0:0:17:0:      class=0xff0000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1ec subvendor=0x17aa subdevice=0x7800
ahci0@pci0:0:17:5:      class=0x010601 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1d2 subvendor=0x17aa subdevice=0x7800
xhci0@pci0:0:20:0:      class=0x0c0330 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1af subvendor=0x17aa subdevice=0x7800
pchtherm0@pci0:0:20:2:  class=0x118000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1b1 subvendor=0x17aa subdevice=0x7800
none6@pci0:0:22:0:      class=0x078000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1ba subvendor=0x17aa subdevice=0x7800
none7@pci0:0:22:1:      class=0x078000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1bb subvendor=0x17aa subdevice=0x7800
none8@pci0:0:22:4:      class=0x078000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1be subvendor=0x17aa subdevice=0x7800
ahci1@pci0:0:23:0:      class=0x010601 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa182 subvendor=0x17aa subdevice=0x7800
pcib1@pci0:0:28:0:      class=0x060400 rev=0xf9 hdr=0x01 vendor=0x8086 device=0xa190 subvendor=0x17aa subdevice=0x7800
isab0@pci0:0:31:0:      class=0x060100 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1c3 subvendor=0x17aa subdevice=0x7800
none9@pci0:0:31:2:      class=0x058000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1a1 subvendor=0x17aa subdevice=0x7800
ichsmb0@pci0:0:31:4:    class=0x0c0500 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1a3 subvendor=0x17aa subdevice=0x7800
none10@pci0:0:31:5:     class=0x0c8000 rev=0x09 hdr=0x00 vendor=0x8086 device=0xa1a4 subvendor=0x17aa subdevice=0x7800
pcib2@pci0:1:0:0:       class=0x060400 rev=0x00 hdr=0x01 vendor=0x19a2 device=0x0120 subvendor=0x0000 subdevice=0x0000
vgapci0@pci0:2:0:0:     class=0x030000 rev=0x42 hdr=0x00 vendor=0x102b device=0x0522 subvendor=0x19a2 subdevice=0x0101
pcib4@pci0:7:2:0:       class=0x060400 rev=0x04 hdr=0x01 vendor=0x8086 device=0x2032 subvendor=0x17aa subdevice=0x7800
none11@pci0:7:5:0:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2034 subvendor=0x17aa subdevice=0x7800
none12@pci0:7:5:2:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2035 subvendor=0x17aa subdevice=0x7800
ioapic1@pci0:7:5:4:     class=0x080020 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2036 subvendor=0x17aa subdevice=0x7800
none13@pci0:7:8:0:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none14@pci0:7:8:1:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none15@pci0:7:8:2:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none16@pci0:7:8:3:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none17@pci0:7:8:4:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none18@pci0:7:8:5:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none19@pci0:7:8:6:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none20@pci0:7:8:7:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none21@pci0:7:9:0:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none22@pci0:7:9:1:      class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208d subvendor=0x17aa subdevice=0x7800
none23@pci0:7:14:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none24@pci0:7:14:1:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none25@pci0:7:14:2:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none26@pci0:7:14:3:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none27@pci0:7:14:4:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none28@pci0:7:14:5:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none29@pci0:7:14:6:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none30@pci0:7:14:7:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none31@pci0:7:15:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none32@pci0:7:15:1:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x208e subvendor=0x17aa subdevice=0x7800
none33@pci0:7:29:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2054 subvendor=0x17aa subdevice=0x7800
none34@pci0:7:29:1:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2055 subvendor=0x17aa subdevice=0x7800
none35@pci0:7:29:2:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2056 subvendor=0x17aa subdevice=0x7800
none36@pci0:7:29:3:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2057 subvendor=0x17aa subdevice=0x7800
none37@pci0:7:30:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2080 subvendor=0x17aa subdevice=0x7800
none38@pci0:7:30:1:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2081 subvendor=0x17aa subdevice=0x7800
none39@pci0:7:30:2:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2082 subvendor=0x17aa subdevice=0x7800
none40@pci0:7:30:3:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2083 subvendor=0x17aa subdevice=0x7800
none41@pci0:7:30:4:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2084 subvendor=0x17aa subdevice=0x7800
none42@pci0:7:30:5:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2085 subvendor=0x17aa subdevice=0x7800
none43@pci0:7:30:6:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2086 subvendor=0x17aa subdevice=0x7800
pcib5@pci0:8:0:0:       class=0x060400 rev=0x09 hdr=0x01 vendor=0x8086 device=0x37c0 subvendor=0xbeef subdevice=0xdead
pcib6@pci0:9:3:0:       class=0x060400 rev=0x09 hdr=0x01 vendor=0x8086 device=0x37c5 subvendor=0xbeef subdevice=0xdead
ixl0@pci0:10:0:0:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
ixl1@pci0:10:0:1:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
ixl2@pci0:10:0:2:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
ixl3@pci0:10:0:3:       class=0x020000 rev=0x09 hdr=0x00 vendor=0x8086 device=0x37d3 subvendor=0x17aa subdevice=0x4021
none44@pci0:90:5:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2034 subvendor=0x17aa subdevice=0x7800
none45@pci0:90:5:2:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2035 subvendor=0x17aa subdevice=0x7800
ioapic2@pci0:90:5:4:    class=0x080020 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2036 subvendor=0x17aa subdevice=0x7800
none46@pci0:90:8:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2066 subvendor=0x17aa subdevice=0x7800
none47@pci0:90:9:0:     class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2066 subvendor=0x17aa subdevice=0x7800
none48@pci0:90:10:0:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2040 subvendor=0x17aa subdevice=0x7800
none49@pci0:90:10:1:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2041 subvendor=0x17aa subdevice=0x7800
none50@pci0:90:10:2:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2042 subvendor=0x17aa subdevice=0x7800
none51@pci0:90:10:3:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2043 subvendor=0x17aa subdevice=0x7800
none52@pci0:90:10:4:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2044 subvendor=0x17aa subdevice=0x7800
none53@pci0:90:10:5:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2045 subvendor=0x17aa subdevice=0x7800
none54@pci0:90:10:6:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2046 subvendor=0x17aa subdevice=0x7800
none55@pci0:90:10:7:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2047 subvendor=0x17aa subdevice=0x7800
none56@pci0:90:11:0:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2048 subvendor=0x17aa subdevice=0x7800
none57@pci0:90:11:1:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2049 subvendor=0x17aa subdevice=0x7800
none58@pci0:90:11:2:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204a subvendor=0x17aa subdevice=0x7800
none59@pci0:90:11:3:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204b subvendor=0x17aa subdevice=0x7800
none60@pci0:90:12:0:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2040 subvendor=0x17aa subdevice=0x7800
none61@pci0:90:12:1:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2041 subvendor=0x17aa subdevice=0x7800
none62@pci0:90:12:2:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2042 subvendor=0x17aa subdevice=0x7800
none63@pci0:90:12:3:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2043 subvendor=0x17aa subdevice=0x7800
none64@pci0:90:12:4:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2044 subvendor=0x17aa subdevice=0x7800
none65@pci0:90:12:5:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2045 subvendor=0x17aa subdevice=0x7800
none66@pci0:90:12:6:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2046 subvendor=0x17aa subdevice=0x7800
none67@pci0:90:12:7:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2047 subvendor=0x17aa subdevice=0x7800
none68@pci0:90:13:0:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2048 subvendor=0x17aa subdevice=0x7800
none69@pci0:90:13:1:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2049 subvendor=0x17aa subdevice=0x7800
none70@pci0:90:13:2:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204a subvendor=0x17aa subdevice=0x7800
none71@pci0:90:13:3:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204b subvendor=0x17aa subdevice=0x7800
pcib9@pci0:173:2:0:     class=0x060400 rev=0x04 hdr=0x01 vendor=0x8086 device=0x2032 subvendor=0x17aa subdevice=0x7800
none72@pci0:173:5:0:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2034 subvendor=0x17aa subdevice=0x7800
none73@pci0:173:5:2:    class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2035 subvendor=0x17aa subdevice=0x7800
ioapic3@pci0:173:5:4:   class=0x080020 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2036 subvendor=0x17aa subdevice=0x7800
none74@pci0:173:14:0:   class=0x110100 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2058 subvendor=0x17aa subdevice=0x7800
none75@pci0:173:14:1:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2059 subvendor=0x17aa subdevice=0x7800
none76@pci0:173:15:0:   class=0x110100 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2058 subvendor=0x17aa subdevice=0x7800
none77@pci0:173:15:1:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2059 subvendor=0x17aa subdevice=0x7800
none78@pci0:173:18:0:   class=0x110100 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204c subvendor=0x17aa subdevice=0x7800
none79@pci0:173:18:1:   class=0x110100 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204d subvendor=0x17aa subdevice=0x7800
none80@pci0:173:18:2:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x204e subvendor=0x17aa subdevice=0x7800
none81@pci0:173:21:0:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2018 subvendor=0x17aa subdevice=0x7800
none82@pci0:173:22:0:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2018 subvendor=0x17aa subdevice=0x7800
none83@pci0:173:22:4:   class=0x088000 rev=0x04 hdr=0x00 vendor=0x8086 device=0x2018 subvendor=0x17aa subdevice=0x7800
mpr0@pci0:174:0:0:      class=0x010700 rev=0x01 hdr=0x00 vendor=0x1000 device=0x00af subvendor=0x1d49 subdevice=0x0200
root@opnsense01:~ #