lvs 一个网卡单个管理ip,多个跨网段VIP解决办法

  1. 云栖社区>
  2. 博客>
  3. 正文

lvs 一个网卡单个管理ip,多个跨网段VIP解决办法

技术小胖子 2017-11-07 17:36:00 浏览1261
展开阅读全文

说明:lvs的vip和realserver的rip是可以跨网段的

ifconfig输出如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
[root@sh-it-prd-lvs01 scripts]# ifconfig 
eth0      Link encap:Ethernet  HWaddr 00:50:56:84:86:FB  
          inet6 addr: fe80::250:56ff:fe84:86fb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7492562 errors:0 dropped:0 overruns:0 frame:0
          TX packets:4443845 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:474201572 (452.2 MiB)  TX bytes:246936351 (235.4 MiB)
 
eth0.200  Link encap:Ethernet  HWaddr 00:50:56:84:86:FB  
          inet6 addr: fe80::250:56ff:fe84:86fb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:12 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:0 (0.0 b)  TX bytes:936 (936.0 b)
 
eth0.1016 Link encap:Ethernet  HWaddr 00:50:56:84:86:FB  
          inet addr:172.24.130.113  Bcast:172.24.130.255  Mask:255.255.255.0
          inet6 addr: fe80::250:56ff:fe84:86fb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:7446938 errors:0 dropped:0 overruns:0 frame:0
          TX packets:3145054 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:367240583 (350.2 MiB)  TX bytes:176803005 (168.6 MiB)
 
eth0.1025 Link encap:Ethernet  HWaddr 00:50:56:84:86:FB  
          inet6 addr: fe80::250:56ff:fe84:86fb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:45600 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1298779 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:2106209 (2.0 MiB)  TX bytes:70132410 (66.8 MiB)
 
lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:16436  Metric:1
          RX packets:1024962 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1024962 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:72580466 (69.2 MiB)  TX bytes:72580466 (69.2 MiB)
 
[root@sh-it-prd-lvs01 scripts]#

其中 eth0.1016 和 eth0.1025做trunk


ip a输出如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
[root@sh-it-prd-lvs01 scripts]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
9: eth0.200@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
11: eth0.1016@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet 172.24.130.113/24 brd 172.24.130.255 scope global eth0.1016
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
12: eth0.1025@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
[root@sh-it-prd-lvs01 scripts]#


最终效果如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
[root@sh-it-prd-lvs01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP qlen 1000
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
9: eth0.200@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
11: eth0.1016@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet 172.24.130.113/24 brd 172.24.130.255 scope global eth0.1016
    inet 172.24.130.5/32 scope global eth0.1016
    inet 172.24.130.6/32 scope global eth0.1016
    inet 172.24.130.7/32 scope global eth0.1016
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
12: eth0.1025@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP 
    link/ether 00:50:56:84:86:fb brd ff:ff:ff:ff:ff:ff
    inet 172.24.25.5/32 scope global eth0.1025
    inet 172.24.25.6/32 scope global eth0.1025
    inet 172.24.25.7/32 scope global eth0.1025
    inet6 fe80::250:56ff:fe84:86fb/64 scope link 
       valid_lft forever preferred_lft forever
[root@sh-it-prd-lvs01 ~]#

keepalived配置文件如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
[root@sh-it-prd-lvs01 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
        notification_email {
        wuxiaoyu@meituan.com
        }
        #notification_email_from root@localhost
        #smtp_server 127.0.0.1
        #smtp_connect_timeout 30
        router_id 980124
}
vrrp_sync_group VG_1 {
        group {
                VI_1
                VI_11
        }
}
vrrp_sync_group VG_2 {
        group {
                VI_2
                VI_21
        }
}
vrrp_sync_group VG_3 {
        group {
                VI_3
                VI_31
        }
}
vrrp_instance VI_1 {
        state MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 50
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.130.5 dev eth0.1016
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
vrrp_instance VI_11 {
        state MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 150
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.25.5 dev eth0.1025
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
virtual_server 172.24.130.5 53 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol UDP
 
        real_server 172.24.130.115 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        MISC_CHECK {
                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.115 53" 
                }
        }
        real_server 172.24.130.116 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        MISC_CHECK {
                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.116 53" 
                }
        }
}
virtual_server 172.24.25.5 53 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol UDP
 
        real_server 172.24.130.115 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        MISC_CHECK {
                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.115 53" 
                }
        }
        real_server 172.24.130.116 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        MISC_CHECK {
                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.116 53" 
                }
        }
}
vrrp_instance VI_2 {
        state  MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 51
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.130.6 dev eth0.1016 
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
vrrp_instance VI_21 {
        state  MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 151
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.25.6 dev eth0.1025
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
virtual_server 172.24.130.6 80 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol TCP
 
        real_server 172.24.130.117 80 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 80
                }
        }
        real_server 172.24.130.118 80 {
        weight 0
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 80
                }
        }
}
virtual_server 172.24.25.6 80 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol TCP
 
        real_server 172.24.130.117 80 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 80
                }
        }
        real_server 172.24.130.118 80 {
        weight 0
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 80
                }
        }
}
vrrp_instance VI_3 {
        state  MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 52
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.130.7 dev eth0.1016 
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
vrrp_instance VI_31 {
        state  MASTER
        interface eth0.1016  ##指定vrrp网卡
        virtual_router_id 152
        priority 150
        nopreempt # no seize,master must add
        advert_int 1
        authentication {
        auth_type PASS
        auth_pass 123456
        }
        virtual_ipaddress {
        172.24.25.7 dev eth0.1025
        }
        notify_master /etc/keepalived/scripts/state_master.sh
        notify_backup /etc/keepalived/scripts/state_backup.sh
        notify_fault  /etc/keepalived/scripts/state_fault.sh
}
virtual_server 172.24.130.7 10051 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol TCP
 
        real_server 1.1.1.1 10051 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 10051
                }
        }
        real_server 1.1.1.2 10051 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 10051
                }
        }
}
virtual_server 172.24.25.7 10051 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol TCP
 
        real_server 1.1.1.1 10051 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 10051
                }
        }
        real_server 1.1.1.2 10051 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
        TCP_CHECK {
                connect_timeout 3
                #nb_get_retry   3
                retry   3
                delay_before_retry 3
                connect_port 10051
                }
        }
}
[root@sh-it-prd-lvs01 ~]#

参考:https://github.com/acassen/keepalived/issues/445



上面的172.24.25.5、172.24.25.6、172.24.25.7 vip  是不通的,解决办法,更新内核参数

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
net.ipv4.ip_forward = 1
 
net.ipv4.conf.default.rp_filter = 1
 
net.ipv4.conf.default.accept_source_route = 0
 
kernel.sysrq = 0
 
kernel.core_uses_pid = 1
 
net.ipv4.tcp_syncookies = 1
 
net.bridge.bridge-nf-call-ip6tables = 0
net.bridge.bridge-nf-call-iptables = 0
net.bridge.bridge-nf-call-arptables = 0
 
kernel.msgmnb = 65536
 
kernel.msgmax = 65536
 
kernel.shmmax = 68719476736
 
kernel.shmall = 4294967296
 
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.all.send_redirects = 0
net.core.netdev_max_backlog = 500000
net.ipv4.conf.all.accept_local=1
net.ipv4.conf.all.rp_filter=2

注意!!!!可以将上面的内容复制到机器上!!!!

eth0.10.25 进包,从eth0.1016转发出包。


另外vip可以和real server ip不在同一个网段,只要有一个物理网卡通接口,keepalived配置文件如下

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
! Configuration File for keepalived
global_defs {
        notification_email {
        #liusichen02@meituan.com
        }
        #notification_email_from root@localhost
        #smtp_server 127.0.0.1
        #smtp_connect_timeout 30
        router_id hwl
}
vrrp_sync_group VG_1 {
        group {
                VI_1
                VI_11
        }
}
vrrp_instance VI_1 {
        state MASTER
        interface eth0.1016
        virtual_router_id 213
        priority 150
        advert_int 1
        authentication {
                auth_type PASS
          auth_pass sankuai
        }
        virtual_ipaddress {
                172.24.130.5
        }
   #     notify_master /etc/keepalived/scripts/state_master.sh
    #    notify_backup /etc/keepalived/scripts/state_backup.sh
    #    notify_fault  /etc/keepalived/scripts/state_fault.sh
}
vrrp_instance VI_11 {
        state MASTER
        interface eth0.1016
        virtual_router_id 214
        priority 150
        advert_int 1
        authentication {
                auth_type PASS
                auth_pass sankuai
        }
        virtual_ipaddress {
                172.24.25.5 dev eth0.1025
                172.24.25.6 dev eth0.1025
                172.24.25.7 dev eth0.1025
        }
    #    notify_master /etc/keepalived/scripts/state_master.sh
    #    notify_backup /etc/keepalived/scripts/state_backup.sh
    #    notify_fault  /etc/keepalived/scripts/state_fault.sh
}
virtual_server 172.24.130.5 53 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        protocol UDP
 
        real_server 172.24.130.115 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
  #      MISC_CHECK {
  #              misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
             #   misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.115 53" 
 #               }
        }
        real_server 172.24.130.116 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
#        MISC_CHECK {
#                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
#                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.116 53#" 
#                }
        }
}
virtual_server 172.24.130.5 53 {
        delay_loop 6
        lb_algo wrr
        lb_kind DR
        persistence_timeout 50
        #sorry_server 127.0.0.1 53
        protocol UDP
 
        real_server 172.24.25.12 53 {
        weight 100
        #notify_up up.sh
        #notify_down down.sh
#        MISC_CHECK {
#                misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
#                misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.115 53" 
#                }
        }
#        real_server 172.24.25.11 53 {
#        weight 100
        #notify_up up.sh
        #notify_down down.sh
 #       MISC_CHECK {
 #               misc_timeout 10
                #connect_timeout 3
                #nb_get_retry   3
                #retry   3
                #delay_before_retry 3
#               misc_path "/usr/local/keepalived/UDP_CHECK.sh 172.24.130.116 53" 
#               }
#       }
}

上述实验证明,一个管理ip多个vip网段也是可以的,





      本文转自Tenderrain 51CTO博客,原文链接:http://blog.51cto.com/tenderrain/1940291,如需转载请自行联系原作者




网友评论

登录后评论
0/500
评论
技术小胖子
+ 关注