我有一台服务器位于两个面向公众的网关后面
INTERNET
| |
GW1 GW2
| |
SERVER
GW1's IP is 192.168.w.x, MAC aa:bb:cc:dd:ee:ff
GW2's IP is 192.168.y.z, MAC ff:ee:dd:cc:bb:aa # note: w != y
我正在尝试负载平衡传出来自服务器的连接。我基于 pepoluan 的解决方案这里以及 mefat 的解决方案这里，但那些是用于处理传入连接的。
这是我的 /etc/iproute2/rt_tables
...
1 gw1
2 gw2
我将 iproute2 路由/规则设置为
ip route add default via 192.168.w.x table gw1
ip route add default via 192.168.y.z table gw2 # w != y
ip rule add fwmark 1 table gw1
ip rule add fwmark 2 table gw2
服务器上的 iptables 规则是
iptables -t mangle -A PREROUTING -j CONNMARK --restore-mark
iptables -t mangle -A PREROUTING -m mark '!' --mark 0 -j ACCEPT
# incoming (ensure that responses are routed back through the GW from which they came)
iptables -t mangle -A PREROUTING -m mac --mac-source aa:bb:cc:dd:ee:ff -j MARK --set-mark 0x1
iptables -t mangle -A PREROUTING -m mac --mac-source ff:ee:dd:cc:bb:aa -j MARK --set-mark 0x2
# outgoing (load balance)
iptables -t mangle -A OUTPUT -m statistic --mode nth --every 2 --packet 0 -j MARK --set-mark 0x1
iptables -t mangle -A OUTPUT -m statistic --mode nth --every 2 --packet 1 -j MARK --set-mark 0x2
# save mark
iptables -t mangle -A POSTROUTING -j CONNMARK --save-mark
我已将私有接口伪装成每个网关上的公共接口，并将服务器作为网关上的受信任的机器，以便网关接受来自它的传入连接。
当我做
ping -m 1 8.8.4.4
在服务器上，我可以看到数据包设置了 0x1 标记，并且它们从 OUTPUT 到达 nat：POSTROUTING，但似乎没有到达网关。
kernel: TRACE: raw:OUTPUT:policy:3 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
kernel: TRACE: mangle:OUTPUT:rule:1 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
kernel: TRACE: mangle:OUTPUT_direct:return:1 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
kernel: TRACE: mangle:OUTPUT:policy:2 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
...
kernel: TRACE: nat:POST_public_allow:return:1 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
kernel: TRACE: nat:POST_public:return:4 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
kernel: TRACE: nat:POSTROUTING:policy:4 IN= OUT=eth0 SRC=192.168.a.b DST=8.8.4.4 LEN=84 TOS=0x00 PREC=0x00 TTL=64 ID=26835 DF PROTO=ICMP TYPE=8 CODE=0 ID=2004 SEQ=1 UID=0 GID=0 MARK=0x1
（“POST_public_allow”之类的链条表明我实际上正在使用firewalld来制定iptables规则。）
我做错了什么或遗漏了什么？我需要 FORWARD 规则吗？如何调试 iproute2 路由/规则？
编辑：根据@AB 的要求添加输出
对于防火墙造成的噪音，我们深表歉意。
[root@server]# ip -br link
lo UNKNOWN 00:00:00:00:00:00 <LOOPBACK,UP,LOWER_UP>
eth0 UP ee:aa:bb:ee:ee:ff <BROADCAST,MULTICAST,UP,LOWER_UP>
[root@server]# ip -4 -br address
lo UNKNOWN 127.0.0.1/8
eth0 UP 192.168.a.b/17 i.j.k.l/24
[root@server]# ip rule
0: from all lookup local
32764: from all fwmark 0x2 lookup gw2
32765: from all fwmark 0x1 lookup gw1
32766: from all lookup main
32767: from all lookup default
[root@server]# ip route
default via i.j.k.1 dev eth0 proto static metric 100
i.j.k.0/24 dev eth0 proto kernel scope link src i.j.k.l metric 100
192.168.128.0/17 dev eth0 proto kernel scope link src 192.168.a.b metric 100
[root@server]# ip route show table gw1
default via 192.168.w.x dev eth0
[root@server]# ip route show table gw2
default via 192.168.y.z dev eth0
[root@server]# iptables-save -c
# Generated by iptables-save v1.6.2 on Wed Jul 11 19:44:43 2018
*nat
:PREROUTING ACCEPT [234747:12721357]
:INPUT ACCEPT [22:1268]
:OUTPUT ACCEPT [855:88393]
:POSTROUTING ACCEPT [526:64778]
:OUTPUT_direct - [0:0]
:POSTROUTING_ZONES - [0:0]
:POSTROUTING_ZONES_SOURCE - [0:0]
:POSTROUTING_direct - [0:0]
:POST_public - [0:0]
:POST_public_allow - [0:0]
:POST_public_deny - [0:0]
:POST_public_log - [0:0]
:POST_trusted - [0:0]
:POST_trusted_allow - [0:0]
:POST_trusted_deny - [0:0]
:POST_trusted_log - [0:0]
:PREROUTING_ZONES - [0:0]
:PREROUTING_ZONES_SOURCE - [0:0]
:PREROUTING_direct - [0:0]
:PRE_public - [0:0]
:PRE_public_allow - [0:0]
:PRE_public_deny - [0:0]
:PRE_public_log - [0:0]
:PRE_trusted - [0:0]
:PRE_trusted_allow - [0:0]
:PRE_trusted_deny - [0:0]
:PRE_trusted_log - [0:0]
[234747:12721357] -A PREROUTING -j PREROUTING_direct
[234747:12721357] -A PREROUTING -j PREROUTING_ZONES_SOURCE
[234747:12721357] -A PREROUTING -j PREROUTING_ZONES
[855:88393] -A OUTPUT -j OUTPUT_direct
[855:88393] -A POSTROUTING -j POSTROUTING_direct
[526:64778] -A POSTROUTING -j POSTROUTING_ZONES_SOURCE
[526:64778] -A POSTROUTING -j POSTROUTING_ZONES
[524:64548] -A POSTROUTING_ZONES -o eth0 -g POST_public
[2:230] -A POSTROUTING_ZONES -g POST_public
[2:230] -A POSTROUTING_ZONES_SOURCE -m set --match-set trusted_hosts dst -j POST_trusted
[329:23615] -A POSTROUTING_direct -m mark ! --mark 0x0 -j SNAT --to-source 192.168.a.b
[526:64778] -A POST_public -j POST_public_log
[526:64778] -A POST_public -j POST_public_deny
[526:64778] -A POST_public -j POST_public_allow
[2:230] -A POST_trusted -j POST_trusted_log
[2:230] -A POST_trusted -j POST_trusted_deny
[2:230] -A POST_trusted -j POST_trusted_allow
[234747:12721357] -A PREROUTING_ZONES -i eth0 -g PRE_public
[0:0] -A PREROUTING_ZONES -g PRE_public
[3:180] -A PREROUTING_ZONES_SOURCE -m set --match-set trusted_hosts src -j PRE_trusted
[234747:12721357] -A PRE_public -j PRE_public_log
[234747:12721357] -A PRE_public -j PRE_public_deny
[234747:12721357] -A PRE_public -j PRE_public_allow
[3:180] -A PRE_trusted -j PRE_trusted_log
[3:180] -A PRE_trusted -j PRE_trusted_deny
[3:180] -A PRE_trusted -j PRE_trusted_allow
COMMIT
# Completed on Wed Jul 11 19:44:43 2018
# Generated by iptables-save v1.6.2 on Wed Jul 11 19:44:43 2018
*mangle
:PREROUTING ACCEPT [623828:32865698]
:INPUT ACCEPT [623836:32866210]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [153676:12802286]
:POSTROUTING ACCEPT [534555:70869287]
:FORWARD_direct - [0:0]
:INPUT_direct - [0:0]
:OUTPUT_direct - [0:0]
:POSTROUTING_direct - [0:0]
:PREROUTING_ZONES - [0:0]
:PREROUTING_ZONES_SOURCE - [0:0]
:PREROUTING_direct - [0:0]
:PRE_public - [0:0]
:PRE_public_allow - [0:0]
:PRE_public_deny - [0:0]
:PRE_public_log - [0:0]
:PRE_trusted - [0:0]
:PRE_trusted_allow - [0:0]
:PRE_trusted_deny - [0:0]
:PRE_trusted_log - [0:0]
[623836:32866210] -A PREROUTING -j PREROUTING_direct
[623828:32865698] -A PREROUTING -j PREROUTING_ZONES_SOURCE
[623828:32865698] -A PREROUTING -j PREROUTING_ZONES
[623836:32866210] -A INPUT -j INPUT_direct
[0:0] -A FORWARD -j FORWARD_direct
[534523:70856073] -A OUTPUT -j OUTPUT_direct
[534555:70869287] -A POSTROUTING -j POSTROUTING_direct
[154527:12862895] -A OUTPUT_direct -j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff
[851:60609] -A OUTPUT_direct -m mark ! --mark 0x0 -j ACCEPT
[76838:6400954] -A OUTPUT_direct -m statistic --mode nth --every 2 --packet 0 -j MARK --set-xmark 0x1/0xffffffff
[76838:6401332] -A OUTPUT_direct -m statistic --mode nth --every 2 --packet 1 -j MARK --set-xmark 0x2/0xffffffff
[534555:70869287] -A POSTROUTING_direct -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff
[623792:32858768] -A PREROUTING_ZONES -i eth0 -g PRE_public
[36:6930] -A PREROUTING_ZONES -g PRE_public
[378266:19709657] -A PREROUTING_ZONES_SOURCE -m set --match-set trusted_hosts src -j PRE_trusted
[623836:32866210] -A PREROUTING_direct -j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff
[8:512] -A PREROUTING_direct -m mark ! --mark 0x0 -j ACCEPT
[0:0] -A PREROUTING_direct -s 192.168.w.x/32 -j MARK --set-xmark 0x1/0xffffffff
[0:0] -A PREROUTING_direct -s 192.168.y.z/32 -j MARK --set-xmark 0x2/0xffffffff
[623828:32865698] -A PRE_public -j PRE_public_log
[623828:32865698] -A PRE_public -j PRE_public_deny
[623828:32865698] -A PRE_public -j PRE_public_allow
[378266:19709657] -A PRE_trusted -j PRE_trusted_log
[378266:19709657] -A PRE_trusted -j PRE_trusted_deny
[378266:19709657] -A PRE_trusted -j PRE_trusted_allow
COMMIT
# Completed on Wed Jul 11 19:44:43 2018
# Generated by iptables-save v1.6.2 on Wed Jul 11 19:44:43 2018
*raw
:PREROUTING ACCEPT [623836:32866210]
:OUTPUT ACCEPT [534530:70860961]
:OUTPUT_direct - [0:0]
:PREROUTING_ZONES - [0:0]
:PREROUTING_ZONES_SOURCE - [0:0]
:PREROUTING_direct - [0:0]
:PRE_public - [0:0]
:PRE_public_allow - [0:0]
:PRE_public_deny - [0:0]
:PRE_public_log - [0:0]
:PRE_trusted - [0:0]
:PRE_trusted_allow - [0:0]
:PRE_trusted_deny - [0:0]
:PRE_trusted_log - [0:0]
[623836:32866210] -A PREROUTING -j PREROUTING_direct
[623836:32866210] -A PREROUTING -j PREROUTING_ZONES_SOURCE
[623836:32866210] -A PREROUTING -j PREROUTING_ZONES
[534530:70860961] -A OUTPUT -j OUTPUT_direct
[623796:32859080] -A PREROUTING_ZONES -i eth0 -g PRE_public
[40:7130] -A PREROUTING_ZONES -g PRE_public
[378268:19709777] -A PREROUTING_ZONES_SOURCE -m set --match-set trusted_hosts src -j PRE_trusted
[623836:32866210] -A PRE_public -j PRE_public_log
[623836:32866210] -A PRE_public -j PRE_public_deny
[623836:32866210] -A PRE_public -j PRE_public_allow
[378268:19709777] -A PRE_trusted -j PRE_trusted_log
[378268:19709777] -A PRE_trusted -j PRE_trusted_deny
[378268:19709777] -A PRE_trusted -j PRE_trusted_allow
COMMIT
# Completed on Wed Jul 11 19:44:43 2018
# Generated by iptables-save v1.6.2 on Wed Jul 11 19:44:43 2018
*security
:INPUT ACCEPT [378299:19711565]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [534530:70860961]
:FORWARD_direct - [0:0]
:INPUT_direct - [0:0]
:OUTPUT_direct - [0:0]
[378299:19711565] -A INPUT -j INPUT_direct
[0:0] -A FORWARD -j FORWARD_direct
[534530:70860961] -A OUTPUT -j OUTPUT_direct
COMMIT
# Completed on Wed Jul 11 19:44:43 2018
# Generated by iptables-save v1.6.2 on Wed Jul 11 19:44:43 2018
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [534519:70858874]
:FORWARD_IN_ZONES - [0:0]
:FORWARD_IN_ZONES_SOURCE - [0:0]
:FORWARD_OUT_ZONES - [0:0]
:FORWARD_OUT_ZONES_SOURCE - [0:0]
:FORWARD_direct - [0:0]
:FWDI_public - [0:0]
:FWDI_public_allow - [0:0]
:FWDI_public_deny - [0:0]
:FWDI_public_log - [0:0]
:FWDI_trusted - [0:0]
:FWDI_trusted_allow - [0:0]
:FWDI_trusted_deny - [0:0]
:FWDI_trusted_log - [0:0]
:FWDO_public - [0:0]
:FWDO_public_allow - [0:0]
:FWDO_public_deny - [0:0]
:FWDO_public_log - [0:0]
:FWDO_trusted - [0:0]
:FWDO_trusted_allow - [0:0]
:FWDO_trusted_deny - [0:0]
:FWDO_trusted_log - [0:0]
:INPUT_ZONES - [0:0]
:INPUT_ZONES_SOURCE - [0:0]
:INPUT_direct - [0:0]
:IN_public - [0:0]
:IN_public_allow - [0:0]
:IN_public_log - [0:0]
:IN_trusted - [0:0]
:IN_trusted_allow - [0:0]
:IN_trusted_deny - [0:0]
:IN_trusted_log - [0:0]
:OUTPUT_direct - [0:0]
[70:20292] -A INPUT -p udp -m multiport --dports 5353 -j ACCEPT
[378201:19689725] -A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
[2:120] -A INPUT -i lo -j ACCEPT
[245557:13155833] -A INPUT -j INPUT_direct
[245557:13155833] -A INPUT -j INPUT_ZONES_SOURCE
[245554:13155653] -A INPUT -j INPUT_ZONES
[10812:434556] -A INPUT -m conntrack --ctstate INVALID -j DROP
[234725:12720089] -A INPUT -j REJECT --reject-with icmp-host-prohibited
[0:0] -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
[0:0] -A FORWARD -i lo -j ACCEPT
[0:0] -A FORWARD -j FORWARD_direct
[0:0] -A FORWARD -j FORWARD_IN_ZONES_SOURCE
[0:0] -A FORWARD -j FORWARD_IN_ZONES
[0:0] -A FORWARD -j FORWARD_OUT_ZONES_SOURCE
[0:0] -A FORWARD -j FORWARD_OUT_ZONES
[0:0] -A FORWARD -m conntrack --ctstate INVALID -j DROP
[0:0] -A FORWARD -j REJECT --reject-with icmp-host-prohibited
[534530:70860961] -A OUTPUT -j OUTPUT_direct
[0:0] -A FORWARD_IN_ZONES -i eth0 -g FWDI_public
[0:0] -A FORWARD_IN_ZONES -g FWDI_public
[0:0] -A FORWARD_IN_ZONES_SOURCE -m set --match-set trusted_hosts src -j FWDI_trusted
[0:0] -A FORWARD_OUT_ZONES -o eth0 -g FWDO_public
[0:0] -A FORWARD_OUT_ZONES -g FWDO_public
[0:0] -A FORWARD_OUT_ZONES_SOURCE -m set --match-set trusted_hosts dst -j FWDO_trusted
[0:0] -A FWDI_public -j FWDI_public_log
[0:0] -A FWDI_public -j FWDI_public_deny
[0:0] -A FWDI_public -j FWDI_public_allow
[0:0] -A FWDI_public -p icmp -j ACCEPT
[0:0] -A FWDI_trusted -j FWDI_trusted_log
[0:0] -A FWDI_trusted -j FWDI_trusted_deny
[0:0] -A FWDI_trusted -j FWDI_trusted_allow
[0:0] -A FWDI_trusted -j ACCEPT
[0:0] -A FWDO_public -j FWDO_public_log
[0:0] -A FWDO_public -j FWDO_public_deny
[0:0] -A FWDO_public -j FWDO_public_allow
[0:0] -A FWDO_trusted -j FWDO_trusted_log
[0:0] -A FWDO_trusted -j FWDO_trusted_deny
[0:0] -A FWDO_trusted -j FWDO_trusted_allow
[0:0] -A FWDO_trusted -j ACCEPT
[245554:13155653] -A INPUT_ZONES -i eth0 -g IN_public
[0:0] -A INPUT_ZONES -g IN_public
[3:180] -A INPUT_ZONES_SOURCE -m set --match-set trusted_hosts src -j IN_trusted
[245554:13155653] -A IN_public -j IN_public_log
[245554:13155653] -A IN_public -j IN_public_deny
[245554:13155653] -A IN_public -j IN_public_allow
[17:1008] -A IN_public -p icmp -j ACCEPT
[0:0] -A IN_public_allow -d 224.0.0.251/32 -p udp -m udp --dport 5353 -m conntrack --ctstate NEW -j ACCEPT
[3:180] -A IN_trusted -j IN_trusted_log
[3:180] -A IN_trusted -j IN_trusted_deny
[3:180] -A IN_trusted -j IN_trusted_allow
[3:180] -A IN_trusted -j ACCEPT
COMMIT
# Completed on Wed Jul 11 19:44:43 2018
答案1
哎呀！我的服务器是裸机，但网关是虚拟专用服务器 (VPS)，并且 VPS 提供商施加了限制，会丢弃不是发往专用网络外机器的数据包。也就是说，由于我的 VPS 提供商施加的限制，我的网络拓扑将无法工作。