DPDK IP 重组 API returns NULL
DPDK IP reassemble API returns NULL
我是 DPDK 的新手，目前正在测试 IP 重组 API，但我遇到了困难。下面是我为测试 IP 重组而编写的 C++ 代码。我从 dpdk 本身提供的示例列表中获取了参考。我正在使用的 dpdk 版本是 debian 机器中的 20.08。 dpdk 用户指南提到 API 适用于 src add、dst add 和数据包 ID，即使所有三个数据都是正确的，但 API returns NULL。任何形式的帮助将不胜感激。提前致谢。
Search the Fragment Table for entry with packet’s <IPv4 Source Address, IPv4 Destination Address, Packet ID>.
If the entry is found, then check if that entry already timed-out. If yes, then free all previously received fragments, and remove information about them from the entry.
If no entry with such key is found, then try to create a new one by one of two ways:
Use as empty entry.
Delete a timed-out entry, free mbufs associated with it mbufs and store a new entry with specified key in it.
Update the entry with new fragment information and check if a packet can be reassembled (the packet’s entry contains all fragments).
If yes, then, reassemble the packet, mark table’s entry as empty and return the reassembled mbuf to the caller.
If no, then return a NULL to the caller.
CONFIG_RTE_LIBRTE_IP_FRAG=y
CONFIG_RTE_LIBRTE_IP_FRAG_DEBUG=y
CONFIG_RTE_LIBRTE_IP_FRAG_MAX_FRAG=100
CONFIG_RTE_LIBRTE_IP_FRAG_TBL_STAT=n
#define DEF_FLOW_NUM 0x1000
#define DEF_FLOW_TTL MS_PER_S
#define IP_FRAG_TBL_BUCKET_ENTRIES 128
static uint32_t max_flow_num = DEF_FLOW_NUM;
static uint32_t max_flow_ttl = DEF_FLOW_TTL;
struct lcore_queue_conf {
struct rte_ip_frag_tbl *frag_tbl;
struct rte_mempool *pool;
struct rte_ip_frag_death_row death_row;
}__rte_cache_aligned;
static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
static inline int setup_queue_tbl(struct lcore_queue_conf *qconf)
{
uint64_t frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S * max_flow_ttl;
qconf->frag_tbl = rte_ip_frag_table_create(max_flow_num, IP_FRAG_TBL_BUCKET_ENTRIES, max_flow_num, frag_cycles, rte_socket_id());
if((qconf->frag_tbl) == NULL){
RTE_LOG(ERR, IP_RSMBL, "Table Failed.");
return -1;
}
qconf->pool=rte_pktmbuf_pool_create("BUFFER", POOL_SIZE*2, POOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
if(qconf->pool== NULL){
RTE_LOG(ERR, IP_RSMBL, "Mem Pool Failed.");
return -1;
}
return 0;
}
static inline void reassemble(struct rte_mbuf *reassemblepkt, struct lcore_queue_conf *qconf, uint64_t cur_tsc)
{
struct rte_mbuf *mo;
struct rte_ether_hdr *eth_hdr;
struct rte_ipv4_hdr *ip_hdr;
eth_hdr = rte_pktmbuf_mtod(reassemblepkt, struct rte_ether_hdr *);
ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
if (rte_ipv4_frag_pkt_is_fragmented(ip_hdr)){
reassemblepkt->l2_len = sizeof(*eth_hdr);
reassemblepkt->l3_len = sizeof(*ip_hdr);
int ip_len;
ip_len = rte_be_to_cpu_16(ip_hdr->total_length);
mo = rte_ipv4_frag_reassemble_packet(qconf->frag_tbl, &qconf->death_row, reassemblepkt, cur_tsc, ip_hdr);
if (mo == NULL){
cout << "Total Length: " << ip_len << ", l3 length: " << reassemblepkt->l3_len << " ,Packet ID: " << ip_hdr->packet_id << " , src add:" << ip_hdr->src_addr << " , dst add:" << ip_hdr->dst_addr << endl;
RTE_LOG(ERR, IP_RSMBL, "Reassemble Failed.\n");
}
if ((mo != reassemblepkt) && (mo != NULL)){
cout << "Reassemble is success." << endl;
reassemblepkt = mo;
}
}
}
static int
lcore_main(struct lcore_queue_conf *qconf)
{
int rx, rec;
struct rte_mbuf *bufs[BUFFER_LENGTH];
uint64_t cur_tsc;
int i;
RTE_ETH_FOREACH_DEV(port){
cout << "RX Thread: Socket ID: " << rte_socket_id() << endl;
cout << "RX Thread: lcore count: " << dec << rte_lcore_count() << endl;
cout << "RX Thread: lcore ID: " << rte_lcore_id() << endl;
cout << "RX Thread Started." << endl;
cout << "=====================================================" << endl;
}
while(!quit_signal) {
cur_tsc = rte_rdtsc();
RTE_ETH_FOREACH_DEV(port){
rx=rte_eth_rx_burst(port, 0, bufs, BUFFER_LENGTH);
if(unlikely(rx == 0 ))
continue;
if(rx){
for(i=0; i<rx; i++)
reassemble(bufs[i], qconf, cur_tsc);
rte_ip_frag_free_death_row(&qconf->death_row, PREFETCH_OFFSET);
rec = rte_ring_enqueue_burst(Myring, (void **)bufs, rx, NULL);
}
}
}
return 0;
}
int main(int argc, char *argv[])
{
int ret;
uint16_t portcheck;
DPDKPORT p1;
struct lcore_queue_conf *qconf;
/* catch ctrl-c so we can print on exit */
signal(SIGINT, int_handler);
/* EAL setup */
ret=rte_eal_init(argc, argv);
cout << "=====================================================" << endl;
if(ret < 0)
cout << "EAL initialising failed." << strerror(-ret) << endl;
else
cout << "EAL initialisation success." << endl;
qconf = &lcore_queue_conf[rte_get_master_lcore()];
if(setup_queue_tbl(qconf) != 0)
rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
.
.
.
.
RTE_ETH_FOREACH_DEV(portcheck){
if(p1.eth_init(portcheck, qconf->pool) != 0)
rte_exit(EXIT_FAILURE, "Ethernet port initialisation failed.");
}
/* Master core call */
lcore_main(qconf);
return 0;
}
输出如下。
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
=====================================================
EAL initialisation success.
USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
PORT 0: Ethernet configuration success.
TX queue configuration success.
RX queue configuration success.
PORT 0: NIC started successfully.
PORT 0: Enabled promiscuous mode.
MAC Addr b4:96:91:3f:21:b6
=====================================================
RX Thread: Socket ID: 0
RX Thread: lcore count: 12
RX Thread: lcore ID: 0
RX Thread Started.
=====================================================
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831203304 , dst add:11304
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
我在 ip_reassemble 示例程序中添加了 printf 并且观察到同样的事情。无法弄清楚为什么会这样。当我在 Wireshark 中打开输入 pcap 时，Wireshark 能够正确地重新组装。请任何人向我推荐一个 pcap 或一种检查 IP 重组是否正常的方法。
/* process this fragment. */
mo = rte_ipv4_frag_reassemble_packet(tbl, dr, m, tms, ip_hdr);
if (mo == NULL) {
/* no packet to send out. */
printf("IP reassemble failed\n");
return;
}
/* we have our packet reassembled. */
if (mo != m) {
printf("IP reassemble success\n");
m = mo;
eth_hdr = rte_pktmbuf_mtod(m,
struct rte_ether_hdr *);
ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
}
root@user:/home/user/dpdk/examples/ip_reassembly# ./build/ip_reassembly -l 1 -- -p 1 -q 2
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
0x7ffdd2cbd50e
IP_RSMBL: Creating LPM table on socket 0
IP_RSMBL: Creating LPM6 table on socket 0
USER1: rte_ip_frag_table_create: allocated of 25165952 bytes at socket 0
Initializing port 0 ... Port 0 modified RSS hash function based on hardware support,requested:0xa38c configured:0x8104
Address:B4:96:91:3F:21:B6
txq=1,0,0
IP_RSMBL: Socket 0: adding route 100.10.0.0/16 (port 0)
IP_RSMBL: Socket 0: adding route 100.20.0.0/16 (port 1)
IP_RSMBL: Socket 0: adding route 100.30.0.0/16 (port 2)
IP_RSMBL: Socket 0: adding route 100.40.0.0/16 (port 3)
IP_RSMBL: Socket 0: adding route 100.50.0.0/16 (port 4)
IP_RSMBL: Socket 0: adding route 100.60.0.0/16 (port 5)
IP_RSMBL: Socket 0: adding route 100.70.0.0/16 (port 6)
IP_RSMBL: Socket 0: adding route 100.80.0.0/16 (port 7)
IP_RSMBL: Socket 0: adding route 0101:0101:0101:0101:0101:0101:0101:0101/48 (port 0)
IP_RSMBL: Socket 0: adding route 0201:0101:0101:0101:0101:0101:0101:0101/48 (port 1)
IP_RSMBL: Socket 0: adding route 0301:0101:0101:0101:0101:0101:0101:0101/48 (port 2)
IP_RSMBL: Socket 0: adding route 0401:0101:0101:0101:0101:0101:0101:0101/48 (port 3)
IP_RSMBL: Socket 0: adding route 0501:0101:0101:0101:0101:0101:0101:0101/48 (port 4)
IP_RSMBL: Socket 0: adding route 0601:0101:0101:0101:0101:0101:0101:0101/48 (port 5)
IP_RSMBL: Socket 0: adding route 0701:0101:0101:0101:0101:0101:0101:0101/48 (port 6)
IP_RSMBL: Socket 0: adding route 0801:0101:0101:0101:0101:0101:0101:0101/48 (port 7)
Checking link status.......................................
done
Port0 Link Up. Speed 10000 Mbps - full-duplex
IP_RSMBL: entering main loop on lcore 1
IP_RSMBL: -- lcoreid=1 portid=0
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
.
.
.
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
^C -- lcoreid=1 portid=0 frag tbl stat:
max entries: 4096;
entries in use: 11;
finds/inserts: 0;
entries added: 0;
entries deleted by timeout: 0;
entries reused by timeout: 0;
total add failures: 0;
add no-space failures: 0;
add hash-collisions failures: 0;
TX bursts: 0
TX packets _queued: 0
TX packets dropped: 0
TX packets send: 0
received signal: 2, exiting
[EDIT-2]：最后一个片段或第一个片段的长度为 0。我不知道这与以下细节有任何关系，只是提一下。
网卡（10GB网卡）
以太网控制器：英特尔公司以太网控制器10G X550T
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
Address sizes: 39 bits physical, 48 bits virtual
CPU(s): 12
On-line CPU(s) list: 0-11
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 1
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 158
Model name: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
Stepping: 10
CPU MHz: 3185.543
CPU max MHz: 3200.0000
CPU min MHz: 800.0000
BogoMIPS: 6384.00
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 12288K
NUMA node0 CPU(s): 0-11
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d
代码的输出..
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
Port 0 MAC: b4 96 91 3f 21 b6
RX Thread: Socket ID: 0
RX Thread: lcore count: 12
RX Thread: lcore ID: 0
mb: 0x172980e80
fp: 0x17e05c5c0
offset: 0
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172980540
fp: 0x17e05c5c0
offset: 24
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x17297fc00
fp: 0x17e05c5c0
offset: 48
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x17297f2c0
fp: 0x17e05c5c0
offset: 72
, IPLen: 24
, ipflag: 8192
idx: 4, frags: 0x4ip_frag_process:145 invalid fragmented packet:
ipv4_frag_pkt: 0x17e05c5c0, key: <ffffffff3b28a8c0, 0xb500>, total_size: 4294967295, frag_size: 96, last_idx: 4
first fragment: ofs: 0, len: 24
last fragment: ofs: 0, len: 0
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b99680
fp: 0x17551e5c0
offset: 96
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98d40
fp: 0x17551e5c0
offset: 120
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98400
fp: 0x17551e5c0
offset: 144
, IPLen: 12
, ipflag: 0
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172c8fb00
fp: 0x17d5885c0
offset: 8
, IPLen: 2028
, ipflag: 0
idx: -1, frags: 0x4ip_frag_process:145 invalid fragmented packet:
ipv4_frag_pkt: 0x17d5885c0, key: <2c28a8c0bbe8, 0x406>, total_size: 2036, frag_size: 4056, last_idx: 2
first fragment: ofs: 0, len: 0
last fragment: ofs: 8, len: 2028
mb_end: (nil)
DPDK API rte_ipv4_frag_reassemble_packet returns NULL in 2 ocassion
- 发生错误
- 尚未收集数据包的所有片段
根据共享的代码和日志，您似乎是
- 多次发送最后一个片段。
- 设置超时为
cur_tsc
注：
- 测试数据包的最简单方法是 运行 对照
ip_reassembly 示例并交叉检查方差。
if (mo == NULL)只表示收不到足够的分片
[edit-1] 因此我要求将您的代码建模为 dpdk 示例 ip_reassembly 因为假设 rte_ipv4_frag_reassemble_packet 返回 NULL 并不总是失败。
[edit-2] 清理代码并添加缺失的库我能够使用正确的片段数据包集来实现它
Reassemble is success.dump mbuf at 0x1736966c0, iova=7b3696740, buf_len=2176
pkt_len=5421, ol_flags=10, nb_segs=4, in_port=0
segment at 0x1736966c0, data=0x1736967c0, data_len=1514
Dump data at [0x1736967c0], len=64
00000000: 00 1D 09 94 65 38 68 5B 35 C0 61 B6 08 00 45 00 | ....e8h[5.a...E.
00000010: 15 1F F5 AF 00 00 40 11 00 00 83 B3 C4 DC 83 B3 | ......@.........
00000020: C4 2E 18 DB 18 DB 15 0B DC E2 06 FD 14 FF 07 29 | ...............)
00000030: 08 07 65 78 61 6D 70 6C 65 08 07 74 65 73 74 41 | ..example..testA
segment at 0x1733a8040, data=0x1733a8162, data_len=1480
segment at 0x1733a8980, data=0x1733a8aa2, data_len=1480
segment at 0x1734dde40, data=0x1734ddf62, data_len=947
[edit-3]
流量生成器：./app/x86_64-native-linuxapp-gcc/pktgen -l 1-4 -- -s 0:test.pcap -P -m [2].0
应用程序：（代码片段编辑并编译为 C 程序）https://pastebin.pl/view/91e533e3
- 构建：
gcc $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --cflags libdpdk) dpdk.c -Wl,-Bstatic $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --libs libdpdk)
- 运行:
sudo LD_LIBRARY_PATH=/path-to-dpdk-sahred-library/ ./a.out
[edit-4] 基于实时调试，问题似乎是连接 2 台机器的集线器交换机。数据包被丢弃或根本不转发。请求建立直接稳定连接以检查@Nirmal 机器上的逻辑。 运行 相同的示例并显示了我机器中的输出。
我是 DPDK 的新手，目前正在测试 IP 重组 API，但我遇到了困难。下面是我为测试 IP 重组而编写的 C++ 代码。我从 dpdk 本身提供的示例列表中获取了参考。我正在使用的 dpdk 版本是 debian 机器中的 20.08。 dpdk 用户指南提到 API 适用于 src add、dst add 和数据包 ID，即使所有三个数据都是正确的，但 API returns NULL。任何形式的帮助将不胜感激。提前致谢。
Search the Fragment Table for entry with packet’s <IPv4 Source Address, IPv4 Destination Address, Packet ID>.
If the entry is found, then check if that entry already timed-out. If yes, then free all previously received fragments, and remove information about them from the entry.
If no entry with such key is found, then try to create a new one by one of two ways:
Use as empty entry.
Delete a timed-out entry, free mbufs associated with it mbufs and store a new entry with specified key in it.
Update the entry with new fragment information and check if a packet can be reassembled (the packet’s entry contains all fragments).
If yes, then, reassemble the packet, mark table’s entry as empty and return the reassembled mbuf to the caller.
If no, then return a NULL to the caller.
CONFIG_RTE_LIBRTE_IP_FRAG=y
CONFIG_RTE_LIBRTE_IP_FRAG_DEBUG=y
CONFIG_RTE_LIBRTE_IP_FRAG_MAX_FRAG=100
CONFIG_RTE_LIBRTE_IP_FRAG_TBL_STAT=n
#define DEF_FLOW_NUM 0x1000
#define DEF_FLOW_TTL MS_PER_S
#define IP_FRAG_TBL_BUCKET_ENTRIES 128
static uint32_t max_flow_num = DEF_FLOW_NUM;
static uint32_t max_flow_ttl = DEF_FLOW_TTL;
struct lcore_queue_conf {
struct rte_ip_frag_tbl *frag_tbl;
struct rte_mempool *pool;
struct rte_ip_frag_death_row death_row;
}__rte_cache_aligned;
static struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
static inline int setup_queue_tbl(struct lcore_queue_conf *qconf)
{
uint64_t frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S * max_flow_ttl;
qconf->frag_tbl = rte_ip_frag_table_create(max_flow_num, IP_FRAG_TBL_BUCKET_ENTRIES, max_flow_num, frag_cycles, rte_socket_id());
if((qconf->frag_tbl) == NULL){
RTE_LOG(ERR, IP_RSMBL, "Table Failed.");
return -1;
}
qconf->pool=rte_pktmbuf_pool_create("BUFFER", POOL_SIZE*2, POOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
if(qconf->pool== NULL){
RTE_LOG(ERR, IP_RSMBL, "Mem Pool Failed.");
return -1;
}
return 0;
}
static inline void reassemble(struct rte_mbuf *reassemblepkt, struct lcore_queue_conf *qconf, uint64_t cur_tsc)
{
struct rte_mbuf *mo;
struct rte_ether_hdr *eth_hdr;
struct rte_ipv4_hdr *ip_hdr;
eth_hdr = rte_pktmbuf_mtod(reassemblepkt, struct rte_ether_hdr *);
ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
if (rte_ipv4_frag_pkt_is_fragmented(ip_hdr)){
reassemblepkt->l2_len = sizeof(*eth_hdr);
reassemblepkt->l3_len = sizeof(*ip_hdr);
int ip_len;
ip_len = rte_be_to_cpu_16(ip_hdr->total_length);
mo = rte_ipv4_frag_reassemble_packet(qconf->frag_tbl, &qconf->death_row, reassemblepkt, cur_tsc, ip_hdr);
if (mo == NULL){
cout << "Total Length: " << ip_len << ", l3 length: " << reassemblepkt->l3_len << " ,Packet ID: " << ip_hdr->packet_id << " , src add:" << ip_hdr->src_addr << " , dst add:" << ip_hdr->dst_addr << endl;
RTE_LOG(ERR, IP_RSMBL, "Reassemble Failed.\n");
}
if ((mo != reassemblepkt) && (mo != NULL)){
cout << "Reassemble is success." << endl;
reassemblepkt = mo;
}
}
}
static int
lcore_main(struct lcore_queue_conf *qconf)
{
int rx, rec;
struct rte_mbuf *bufs[BUFFER_LENGTH];
uint64_t cur_tsc;
int i;
RTE_ETH_FOREACH_DEV(port){
cout << "RX Thread: Socket ID: " << rte_socket_id() << endl;
cout << "RX Thread: lcore count: " << dec << rte_lcore_count() << endl;
cout << "RX Thread: lcore ID: " << rte_lcore_id() << endl;
cout << "RX Thread Started." << endl;
cout << "=====================================================" << endl;
}
while(!quit_signal) {
cur_tsc = rte_rdtsc();
RTE_ETH_FOREACH_DEV(port){
rx=rte_eth_rx_burst(port, 0, bufs, BUFFER_LENGTH);
if(unlikely(rx == 0 ))
continue;
if(rx){
for(i=0; i<rx; i++)
reassemble(bufs[i], qconf, cur_tsc);
rte_ip_frag_free_death_row(&qconf->death_row, PREFETCH_OFFSET);
rec = rte_ring_enqueue_burst(Myring, (void **)bufs, rx, NULL);
}
}
}
return 0;
}
int main(int argc, char *argv[])
{
int ret;
uint16_t portcheck;
DPDKPORT p1;
struct lcore_queue_conf *qconf;
/* catch ctrl-c so we can print on exit */
signal(SIGINT, int_handler);
/* EAL setup */
ret=rte_eal_init(argc, argv);
cout << "=====================================================" << endl;
if(ret < 0)
cout << "EAL initialising failed." << strerror(-ret) << endl;
else
cout << "EAL initialisation success." << endl;
qconf = &lcore_queue_conf[rte_get_master_lcore()];
if(setup_queue_tbl(qconf) != 0)
rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
.
.
.
.
RTE_ETH_FOREACH_DEV(portcheck){
if(p1.eth_init(portcheck, qconf->pool) != 0)
rte_exit(EXIT_FAILURE, "Ethernet port initialisation failed.");
}
/* Master core call */
lcore_main(qconf);
return 0;
}
输出如下。
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
=====================================================
EAL initialisation success.
USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
PORT 0: Ethernet configuration success.
TX queue configuration success.
RX queue configuration success.
PORT 0: NIC started successfully.
PORT 0: Enabled promiscuous mode.
MAC Addr b4:96:91:3f:21:b6
=====================================================
RX Thread: Socket ID: 0
RX Thread: lcore count: 12
RX Thread: lcore ID: 0
RX Thread Started.
=====================================================
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831203304 , dst add:11304
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 40960 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 2048, l3 length: 20 ,Packet ID: 1030 , src add:2831191337 , dst add:10280
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 44, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
IP_RSMBL: Reassemble Failed.
Total Length: 32, l3 length: 20 ,Packet ID: 42752 , src add:992520384 , dst add:4294967295
我在 ip_reassemble 示例程序中添加了 printf 并且观察到同样的事情。无法弄清楚为什么会这样。当我在 Wireshark 中打开输入 pcap 时，Wireshark 能够正确地重新组装。请任何人向我推荐一个 pcap 或一种检查 IP 重组是否正常的方法。
/* process this fragment. */
mo = rte_ipv4_frag_reassemble_packet(tbl, dr, m, tms, ip_hdr);
if (mo == NULL) {
/* no packet to send out. */
printf("IP reassemble failed\n");
return;
}
/* we have our packet reassembled. */
if (mo != m) {
printf("IP reassemble success\n");
m = mo;
eth_hdr = rte_pktmbuf_mtod(m,
struct rte_ether_hdr *);
ip_hdr = (struct rte_ipv4_hdr *)(eth_hdr + 1);
}
root@user:/home/user/dpdk/examples/ip_reassembly# ./build/ip_reassembly -l 1 -- -p 1 -q 2
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
0x7ffdd2cbd50e
IP_RSMBL: Creating LPM table on socket 0
IP_RSMBL: Creating LPM6 table on socket 0
USER1: rte_ip_frag_table_create: allocated of 25165952 bytes at socket 0
Initializing port 0 ... Port 0 modified RSS hash function based on hardware support,requested:0xa38c configured:0x8104
Address:B4:96:91:3F:21:B6
txq=1,0,0
IP_RSMBL: Socket 0: adding route 100.10.0.0/16 (port 0)
IP_RSMBL: Socket 0: adding route 100.20.0.0/16 (port 1)
IP_RSMBL: Socket 0: adding route 100.30.0.0/16 (port 2)
IP_RSMBL: Socket 0: adding route 100.40.0.0/16 (port 3)
IP_RSMBL: Socket 0: adding route 100.50.0.0/16 (port 4)
IP_RSMBL: Socket 0: adding route 100.60.0.0/16 (port 5)
IP_RSMBL: Socket 0: adding route 100.70.0.0/16 (port 6)
IP_RSMBL: Socket 0: adding route 100.80.0.0/16 (port 7)
IP_RSMBL: Socket 0: adding route 0101:0101:0101:0101:0101:0101:0101:0101/48 (port 0)
IP_RSMBL: Socket 0: adding route 0201:0101:0101:0101:0101:0101:0101:0101/48 (port 1)
IP_RSMBL: Socket 0: adding route 0301:0101:0101:0101:0101:0101:0101:0101/48 (port 2)
IP_RSMBL: Socket 0: adding route 0401:0101:0101:0101:0101:0101:0101:0101/48 (port 3)
IP_RSMBL: Socket 0: adding route 0501:0101:0101:0101:0101:0101:0101:0101/48 (port 4)
IP_RSMBL: Socket 0: adding route 0601:0101:0101:0101:0101:0101:0101:0101/48 (port 5)
IP_RSMBL: Socket 0: adding route 0701:0101:0101:0101:0101:0101:0101:0101/48 (port 6)
IP_RSMBL: Socket 0: adding route 0801:0101:0101:0101:0101:0101:0101:0101/48 (port 7)
Checking link status.......................................
done
Port0 Link Up. Speed 10000 Mbps - full-duplex
IP_RSMBL: entering main loop on lcore 1
IP_RSMBL: -- lcoreid=1 portid=0
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
.
.
.
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
IP reassemble failed
^C -- lcoreid=1 portid=0 frag tbl stat:
max entries: 4096;
entries in use: 11;
finds/inserts: 0;
entries added: 0;
entries deleted by timeout: 0;
entries reused by timeout: 0;
total add failures: 0;
add no-space failures: 0;
add hash-collisions failures: 0;
TX bursts: 0
TX packets _queued: 0
TX packets dropped: 0
TX packets send: 0
received signal: 2, exiting
[EDIT-2]：最后一个片段或第一个片段的长度为 0。我不知道这与以下细节有任何关系，只是提一下。
网卡（10GB网卡） 以太网控制器：英特尔公司以太网控制器10G X550T
Architecture: x86_64
CPU op-mode(s): 32-bit, 64-bit
Byte Order: Little Endian
Address sizes: 39 bits physical, 48 bits virtual
CPU(s): 12
On-line CPU(s) list: 0-11
Thread(s) per core: 2
Core(s) per socket: 6
Socket(s): 1
NUMA node(s): 1
Vendor ID: GenuineIntel
CPU family: 6
Model: 158
Model name: Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz
Stepping: 10
CPU MHz: 3185.543
CPU max MHz: 3200.0000
CPU min MHz: 800.0000
BogoMIPS: 6384.00
Virtualization: VT-x
L1d cache: 32K
L1i cache: 32K
L2 cache: 256K
L3 cache: 12288K
NUMA node0 CPU(s): 0-11
Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx rdseed adx smap clflushopt intel_pt xsaveopt xsavec xgetbv1 xsaves dtherm arat pln pts hwp hwp_notify hwp_act_window hwp_epp md_clear flush_l1d
代码的输出..
EAL: Detected 12 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'VA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Invalid NUMA socket, default to 0
EAL: Invalid NUMA socket, default to 0
EAL: using IOMMU type 1 (Type 1)
EAL: Probe PCI driver: net_ixgbe (8086:15d1) device: 0000:01:00.0 (socket 0)
EAL: Invalid NUMA socket, default to 0
EAL: No legacy callbacks, legacy socket not created
USER1: rte_ip_frag_table_create: allocated of 201326720 bytes at socket 0
Port 0 MAC: b4 96 91 3f 21 b6
RX Thread: Socket ID: 0
RX Thread: lcore count: 12
RX Thread: lcore ID: 0
mb: 0x172980e80
fp: 0x17e05c5c0
offset: 0
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172980540
fp: 0x17e05c5c0
offset: 24
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x17297fc00
fp: 0x17e05c5c0
offset: 48
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x17297f2c0
fp: 0x17e05c5c0
offset: 72
, IPLen: 24
, ipflag: 8192
idx: 4, frags: 0x4ip_frag_process:145 invalid fragmented packet:
ipv4_frag_pkt: 0x17e05c5c0, key: <ffffffff3b28a8c0, 0xb500>, total_size: 4294967295, frag_size: 96, last_idx: 4
first fragment: ofs: 0, len: 24
last fragment: ofs: 0, len: 0
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b99680
fp: 0x17551e5c0
offset: 96
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98d40
fp: 0x17551e5c0
offset: 120
, IPLen: 24
, ipflag: 8192
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172b98400
fp: 0x17551e5c0
offset: 144
, IPLen: 12
, ipflag: 0
mb_end: (nil)
ERR, IP_RSMBL, Reassemble Failed.
mb: 0x172c8fb00
fp: 0x17d5885c0
offset: 8
, IPLen: 2028
, ipflag: 0
idx: -1, frags: 0x4ip_frag_process:145 invalid fragmented packet:
ipv4_frag_pkt: 0x17d5885c0, key: <2c28a8c0bbe8, 0x406>, total_size: 2036, frag_size: 4056, last_idx: 2
first fragment: ofs: 0, len: 0
last fragment: ofs: 8, len: 2028
mb_end: (nil)
DPDK API rte_ipv4_frag_reassemble_packet returns NULL in 2 ocassion
- 发生错误
- 尚未收集数据包的所有片段
根据共享的代码和日志，您似乎是
- 多次发送最后一个片段。
- 设置超时为
cur_tsc
注：
- 测试数据包的最简单方法是 运行 对照
ip_reassembly示例并交叉检查方差。 if (mo == NULL)只表示收不到足够的分片
[edit-1] 因此我要求将您的代码建模为 dpdk 示例 ip_reassembly 因为假设 rte_ipv4_frag_reassemble_packet 返回 NULL 并不总是失败。
[edit-2] 清理代码并添加缺失的库我能够使用正确的片段数据包集来实现它
Reassemble is success.dump mbuf at 0x1736966c0, iova=7b3696740, buf_len=2176
pkt_len=5421, ol_flags=10, nb_segs=4, in_port=0
segment at 0x1736966c0, data=0x1736967c0, data_len=1514
Dump data at [0x1736967c0], len=64
00000000: 00 1D 09 94 65 38 68 5B 35 C0 61 B6 08 00 45 00 | ....e8h[5.a...E.
00000010: 15 1F F5 AF 00 00 40 11 00 00 83 B3 C4 DC 83 B3 | ......@.........
00000020: C4 2E 18 DB 18 DB 15 0B DC E2 06 FD 14 FF 07 29 | ...............)
00000030: 08 07 65 78 61 6D 70 6C 65 08 07 74 65 73 74 41 | ..example..testA
segment at 0x1733a8040, data=0x1733a8162, data_len=1480
segment at 0x1733a8980, data=0x1733a8aa2, data_len=1480
segment at 0x1734dde40, data=0x1734ddf62, data_len=947
[edit-3]
流量生成器：./app/x86_64-native-linuxapp-gcc/pktgen -l 1-4 -- -s 0:test.pcap -P -m [2].0
应用程序：（代码片段编辑并编译为 C 程序）https://pastebin.pl/view/91e533e3
- 构建：
gcc $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --cflags libdpdk) dpdk.c -Wl,-Bstatic $(PKG_CONFIG_PATH=[path-to-dpdk-pkgconfig] pkg-config --static --libs libdpdk) - 运行:
sudo LD_LIBRARY_PATH=/path-to-dpdk-sahred-library/ ./a.out
[edit-4] 基于实时调试，问题似乎是连接 2 台机器的集线器交换机。数据包被丢弃或根本不转发。请求建立直接稳定连接以检查@Nirmal 机器上的逻辑。 运行 相同的示例并显示了我机器中的输出。