The document provides detailed instructions for enabling various features and functionalities on Mellanox network adapters, including SR-IOV, RoCEv2, hardware offloads, and integration with Open vSwitch, across multiple hardware models like ConnectX-4 and XL710. It includes command-line steps for configuring the system, installing necessary software packages, and verifying configurations. Overall, the document serves as a comprehensive guide for networking professionals to optimize Mellanox adapters for enhanced performance and capabilities.

1. Network Adapter Deep dive
6th May, 2020 SAKURA Internet, Inc. Research Center SR / Naoto MATSUMOTO
(C) Copyright 1996-2020 SAKURA Internet Inc

2. Enable SR-IOV TIPS (Melalnox ConnectX-4 EN)
2
# grep IOMMU /boot/config-`uname -r` | grep INTEL
CONFIG_INTEL_IOMMU=y
# vi /etc/default/grub
GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt"
# update-grub
# cd /opt
# tar xzvf ./mlnx-en-4.4-2.0.7.0-ubuntu16.04-x86_64.tgz
# cd mlnx-en-4.4-2.0.7.0-ubuntu16.04-x86_64
# ./install
# cd /opt
# tar xzvf ./mft-4.10.0-104-x86_64-deb.tgz
# cd mft-4.10.0-104-x86_64-deb
# ./install.sh
# mst start
# mst status
# mlxconfig -d /dev/mst/mt4115_pciconf0 set SRIOV_EN=1 NUM_OF_VFS=4
# sync; sync; sync; reboot
# echo 4 > /sys/class/net/enp101s0/device/sriov_numvfs
# dmesg | grep IOMMU
# mlxconfig -d /dev/mst/mt4115_pciconf0 q
# lspci -D | grep Mellanox
# lshw -class network -businfo
Bus info Device Class Description
========================================================
pci@0000:65:00.0 enp101s0 network MT27700 Family [ConnectX-4]
pci@0000:65:00.1 enp101s0f1 network Illegal Vendor ID
pci@0000:65:00.2 enp101s0f2 network Illegal Vendor ID
pci@0000:65:00.3 enp101s0f3 network Illegal Vendor ID
pci@0000:65:00.4 enp101s0f4 network Illegal Vendor ID
SOURCE: SAKURA Internet Research Center (2020/05)

3. Enable SR-IOV TIPS (Intel XL710)
3
# grep IOMMU /boot/config-`uname -r` | grep INTEL
CONFIG_INTEL_IOMMU=y
# vi /etc/default/grub
GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt"
# update-grub
# sync; sync; sync; reboot
# rmmod i40e
# modprobe i40e max_vfs=4
# echo 4 > /sys/class/net/enp101s0/device/sriov_numvfs
# lscpi
# lshw -class network -businfo
Bus info Device Class Description
========================================================
pci@0000:65:00.0 enp101s0 network Ethernet Controller XL710 for 40GbE QSFP+
pci@0000:65:02.0 enp101s2 network Illegal Vendor ID
pci@0000:65:02.1 enp101s2f1 network Illegal Vendor ID
pci@0000:65:02.2 enp101s2f2 network Illegal Vendor ID
pci@0000:65:02.3 enp101s2f3 network Illegal Vendor ID
SOURCE: SAKURA Internet Research Center (2020/05)

4. Enable RoCEv2 TIPS (Mellanox Connect X5)
4
# cd /opt
# tar xzvf ./mlnx-en-4.4-2.0.7.0-ubuntu18.04-x86_64.tgz
# cd mlnx-en-4.4-2.0.7.0-ubuntu18.04-x86_64
# ./insall
# /etc/init.d/mlnx-en.d restart
# cd /opt
# tar xzvf ./MLNX_OFED_LINUX-4.4-2.0.7.0-ubuntu18.04-x86_64.tgz
# cd MLNX_OFED_LINUX-4.4-2.0.7.0-ubuntu18.04-x86_64
# ./mlnxofedinstall
# /etc/init.d/openibd restart
# ufw disable
# ethtool -i enp1s0
# cma_roce_mode -d mlx5_0 -p 1 -m 2
RoCE v2
# show_gids
# mlnx_qos -i enp1s0 --pfc 0,0,0,1,0,0,0,0
# sysctl -w net.ipv4.tcp_ecn=1
# cma_roce_tos -d mlx5_0 -t 24
# /etc/init.d/irqbalance stop
# mlnx_affinity start
# ethtool -K enp1s0 tx-nocache-copy off
# echo performance > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
SERVER# ifconfig enp1s0 1.1.1.1/24 up up
CLIENT# ifconfig enp1s0 1.1.1.2/24 up up
SERVER# ib_write_bw -R --report_gbits --port=12500 -D 10
CLIENT# ib_write_bw -R --report_gbits 1.1.1.1 --port=12500 -D 10
SERVER# ib_write_lat -R --port=12500 -D 10
CLIENT# ib_write_lat -R 1.1.1.1 --port=12500 -D 10
SOURCE: SAKURA Internet Research Center (2020/05)
https://community.mellanox.com/docs/DOC-2650
https://community.mellanox.com/docs/DOC-2671
https://community.mellanox.com/docs/DOC-2521
https://community.mellanox.com/docs/DOC-2733

5. Enable TC H/W Offload TIPS (Mellanox Connect X5)
5
# apt install -y elfutils libelf-dev libmnl-dev bison flex pkg-config
# cd /opt; git clone https://git.kernel.org/pub/scm/network/iproute2/iproute2-next.git
# cd iproute2-next; ./configure ; make ; make install
# cd /opt; wget http://www.mellanox.com/downloads/ofed/MLNX_EN-4.4-2.0.7.0/mlnx-en-4.4-2.0.7.0-ubuntu18.04-x86_64.tgz
# tar xzvf mlnx-en-4.4-2.0.7.0-ubuntu18.04-x86_64.tgz; cd mlnx-en-4.4-2.0.7.0-ubuntu18.04-x86_64; ./install
# /etc/init.d/mlnx-en.d restart
# ethtool -k enp101s0 | grep hw-tc
hw-tc-offload: on
# grep IOMMU /boot/config-`uname -r` | grep INTEL
CONFIG_INTEL_IOMMU=y
# vi /etc/default/grub
GRUB_CMDLINE_LINUX="intel_iommu=on iommu=pt"
# update-grub
# dmesg | grep IOMMU
# cd /opt; tar xzvf ./mft-4.10.0-104-x86_64-deb.tgz; cd mft-4.10.0-104-x86_64-deb; ./install.sh
# mst start
# mlxconfig -d /dev/mst/mt4115_pciconf0 set SRIOV_EN=1 NUM_OF_VFS=4
# sync; sync; sync; reboot
# echo 4 > /sys/class/net/enp101s0/device/sriov_numvfs
# mlxconfig -d /dev/mst/mt4115_pciconf0 q
# lspci -D | grep Mellanox
# lshw -class network -businfo
# tc qdisc add dev enp101s0 ingress
# tc filter add dev enp101s0 protocol ip parent ffff: flower skip_sw ip_proto tcp dst_port 80 action drop
# tc -s filter show dev enp101s0 ingress
# tc qdisc add dev enp101s0f1 ingress
# tc -s monitor
# tc action help
# tc qdisc help
SOURCE: SAKURA Internet Research Center (2020/05)

6. Enable eswtich/ASAP2 TIPS (Mellanox Connect X5)
6
# mst start
# mlxconfig -d /dev/mst/mt4121_pciconf0 query
# mlxconfig -d /dev/mst/mt4121_pciconf0 set SRIOV_EN=1
# mlxconfig -d /dev/mst/mt4121_pciconf0 set NUM_OF_VFS=32
# sync; sync; sync; reboot
# echo 1 > /sys/class/net/enp1s0f0/device/sriov_numvfs
# devlink dev show
pci/0000:01:00.0 (*PF enp1s0f0 )
pci/0000:01:00.1 (*VF enp1s0f1 )
# echo 0000:01:00.1 > /sys/bus/pci/drivers/mlx5_core/unbind
# devlink dev eswitch set pci/0000:01:00.0 mode switchdev
# echo 0000:01:00.1 > /sys/bus/pci/drivers/mlx5_core/bind
# apt install openvswitch-switch -y
# /etc/init.d/openvswitch-switch start
# ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
# /etc/init.d/openvswitch-switch restart
# ovs-vsctl add-br ovs-sriov
# ovs-vsctl add-port ovs-sriov enp1s0f0
# ovs-vsctl add-port ovs-sriov enp1s0f0_0
# ifconfig enp1s0f0 up up (*PF)
# ifconfig enp1s0f0_0 up up (*VF representor)
# ip netns add TEST (*namespace TEST)
# ip link set enp1s0f1 netns TEST
# ip netns exec TEST ifconfig enp1s0f1 up up (*VF)
# ip netns exec TEST dhcient enp1s0f1 (*VF assigned ip address 1.2.3.4)
# ip netns exec TEST ping 8.8.8.8
# ovs-dpctl dump-flows type=offloaded
# devlink dev eswitch show pci/0000:01:00.0
SOURCE: SAKURA Internet Research Center (2020/05)

7. Enable FD.io VPP TIPS (Intel XL710)
7
# update-grub
# apt remove --purge vpp*
# vi /etc/apt/sources.list.d/99fd.io.list
deb [trusted=yes] https://nexus.fd.io/content/repositories/fd.io.stable.1807.ubuntu.xenial.main/ ./
# apt update
# apt dist-upgrade -y
# apt install -y vpp vpp-lib vpp-plugins vpp-dpdk-dkms
# lshw -class network -businfo
# ifconfig enp101s0 down
# vi /etc/vpp/startup.conf
dpdk {
dev 0000:65:00.0
}
# service vpp restart
# vppctl show pci
Address Sock VID:PID Link Speed Driver Product Name
0000:65:00.0 0 8086:1584 8.0 GT/s x8 uio_pci_generic XL710 40GbE
# vppctl set int ip address FortyGigabitEthernet65/0/0 1.2.3.4/24
# vppctl set int state FortyGigabitEthernet65/0/0 up
# vppctl show interface addr
FortyGigabitEthernet65/0/0 (up):
L3 1.2.3.4/24
local0 (dn):
# vppctl show version
vpp v18.07-rc2~6-gdb6d6b3~b28 built by root on 10268b67c8b1 ...
SOURCE: SAKURA Internet Research Center (2020/05)

8. Enable L4LB eBDF/XDP H/W Offload TIPS (Agilio CX SmartNIC)
8
# wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v4.19-rc3/linux-headers-4.19.0-041900rc3...201809120832_all.deb
# wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v4.19-rc3/linux-headers-4.19.0-041900rc3...201809120832_amd64.deb
# wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v4.19-rc3/linux-modules-4.19.0-041900rc3...201809120832_amd64.deb
# wget http://kernel.ubuntu.com/~kernel-ppa/mainline/v4.19-rc3/linux-image-unsigned-4.19.0-041900rc3...20832_amd64.deb
# dpkg -i linux-headers-4.19.0-041900rc3_4.19.0-041900rc3.201809120832_all.deb
# dpkg -i linux-headers-4.19.0-041900rc3-generic_4.19.0-041900rc3.201809120832_amd64.deb
# dpkg -i linux-modules-4.19.0-041900rc3-generic_4.19.0-041900rc3.201809120832_amd64.deb
# dpkg -i linux-image-unsigned-4.19.0-041900rc3-generic_4.19.0-041900rc3.201809120832_amd64.deb
# sync; sync; sync; reboot
# dpkg -i agilio-bpf-firmware-2.0.6.121-1.deb
# dpkg -i bpftool-4.18_amd64.deb
# modprobe -r nfp; modprobe nfp
# ethtool -k enp101s0np0
# cd /opt; apt install elfutils libelf-dev libmnl-dev bison flex pkg-config
# git clone https://git.kernel.org/pub/scm/network/iproute2/iproute2-next.git
# cd iproute2-next; ./configure ; make; make install
# cd /opt; apt install -y clang llvm ; git clone https://github.com/Netronome/bpf-samples.git
# cd bpf-samples/l4lb/; make
# ip link set dev enp101s0np0 xdpoffload obj l4lb_xdp.o sec xdp
# bpftool prog show
12: xdp tag 803980e02b8beb36 dev enp101s0np0
loaded_at 2018-09-13T12:47:36+0900 uid 0
xlated 1784B jited 4024B memlock 4096B map_ids 12
# ./l4lb_map.py -i enp101s0np0 -f ./destination_samples/32_destinations.csv
# ./l4lb_stats.py -i enp101s0np0
== Load balancer outbound statistics [Offload] ==
1 10.0.0.57 0 pkts/s 0 bits/s
[Totals] 0 pkts/s 0 bits/s
SOURCE: SAKURA Internet Research Center (2020/05)