实验拓扑:
实验准备:
CentOS6.5-x86-64
node1 10.0.0.11 软件:ipvsadm+keepalived 角色:Director1 node2 10.0.0.12 软件:ipvsadm+keepalived 角色:Director2 node3 10.0.0.13 软件:nginx 角色:Realerver1 node4 10.0.0.14 软件:nginx 角色:Realerver2 |
一、在两台Director上安装配置ipvsadm和keepalived
1、安装ipvsadm
1
2
3
|
# yum install ipvsadm -y
# ipvsadm -v
ipvsadm v1.26 2008
/5/15
(compiled with popt and IPVS v1.2.1)
|
2、安装keepalived
1
2
3
4
5
6
7
8
9
10
11
12
13
|
# sh hm-keepalived-install.sh (keepalived安装脚本)
#!/bin/bash
# Author: hm Email: mail@huangming.org
kernel=
/usr/src/kernels/
$(
uname
-r)
yum
install
gcc gcc-c++ pcre-devel openssl-devel popt-devel libnl-devel libnfnetlink libnfnetlink-devel -y
[ -e keepalived-1.2.23.
tar
.gz ] || wget http:
//www
.keepalived.org
/software/keepalived-1
.2.23.
tar
.gz
tar
-zxf keepalived-1.2.23.
tar
.gz &&
cd
keepalived-1.2.23 &&
.
/configure
--prefix=
/usr/local/keepalived
--sysconf=
/etc
--with-kernel-
dir
=${kernel}
make
&&
make
install
chkconfig --add keepalived && chkconfig keepalived on
ln
-s
/usr/local/keepalived/sbin/keepalived
/usr/sbin/
|
三、在两台Realserver上安装配置nginx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
# sh hm-nginx-install.sh(nginx安装脚本,只需将nginx源码包放到/usr/local/src目录下,执行即可)
#!/bin/bash
# Author: hm Email: mail@huangming.org
nginx_s=
/usr/local/src/nginx-1
.8.1.
tar
.gz
nginx_v=nginx-1.8.1
for
p
in
gcc gcc-c++ zlib pcre pcre-devel openssl openssl-devel
do
if
! rpm -qa |
grep
-q
"^$p"
;
then
yum
install
-y $p
fi
done
if
[ ! -d
/etc/nginx
-o ! -e
/usr/sbin/nginx
];
then
useradd
-s
/sbin/nologin
nginx -M
cd
/usr/local/src
&&
tar
-zxf ${nginx_s} -C . &&
cd
$nginx_v &&
.
/configure
\
--user=nginx \
--group=nginx \
--prefix=
/usr/local/nginx
\
--sbin-path=
/usr/sbin/nginx
\
--conf-path=
/etc/nginx/nginx
.conf \
--pid-path=
/var/run/nginx
.pid \
--lock-path=
/var/lock/subsys/nginx
\
--with-http_stub_status_module \
--with-http_ssl_module \
--with-http_gzip_static_module \
--http-log-path=
/var/log/nginx/access
.log \
--error-log-path=
/var/log/nginx/error
.log \
--http-client-body-temp-path=
/tmp/nginx/client_body
\
--http-proxy-temp-path=
/tmp/nginx/proxy
\
--http-fastcgi-temp-path=
/tmp/nginx/fastcgi
\
--with-http_degradation_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-pcre
make
&&
make
install
chown
-R nginx:nginx
/usr/local/nginx
&&
mkdir
/tmp/nginx/client_body
-p &&
mkdir
/etc/nginx/vhosts
# nginx.conf
cat
>
/etc/nginx/nginx
.conf << EOF
user nginx nginx;
worker_processes $(
cat
/proc/cpuinfo
|
grep
'processor'
|
wc
-l);
error_log
/var/log/nginx/error
.log notice;
pid
/var/run/nginx
.pid;
worker_rlimit_nofile 65535;
events
{
use epoll;
worker_connections 65535;
}
http
{
include mime.types;
default_type application
/octet-stream
;
server_names_hash_bucket_size 2048;
server_names_hash_max_size 2048;
log_format main
'\$remote_addr - \$remote_user [\$time_local] "\$request" '
'\$status \$body_bytes_sent "\$http_referer" "\$request_time"'
'"\$http_user_agent" \$HTTP_X_Forwarded_For'
;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 60;
client_header_timeout 3m;
client_body_timeout 3m;
client_max_body_size 10m;
client_body_buffer_size 256k;
send_timeout 3m;
connection_pool_size 256;
client_header_buffer_size 32k;
large_client_header_buffers 4 64k;
request_pool_size 4k;
output_buffers 4 32k;
postpone_output 1460;
client_body_temp_path
/tmp/nginx/client_body
;
proxy_temp_path
/tmp/nginx/proxy
;
fastcgi_temp_path
/tmp/nginx/fastcgi
;
gzip
on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_comp_level 3;
gzip_http_version 1.1;
gzip_types text
/plain
application
/x-javascript
text
/css
text
/htm
application
/xml
;
gzip_vary on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
fastcgi_intercept_errors on;
include vhosts/*.conf;
}
EOF
# nginx_vhosts.conf
cat
>
/etc/nginx/vhosts/www
.conf << EOF
server {
listen 80;
server_name $(
ifconfig
eth0 |
awk
-F
"[ :]+"
'/inet addr/{print $4}'
);
index index.html index.htm index.php index.jsp;
charset UTF8;
root
/data/www/html
;
access_log
/var/log/nginx/access
.log main;
}
EOF
mkdir
/data/www/html
-p &&
echo
$(
ifconfig
eth0 |
awk
-F
"[ :]+"
'/inet addr/{print $4}'
) >
/data/www/html/index
.html
/usr/sbin/nginx
-t &&
/usr/sbin/nginx
echo
echo
"--------------------------------"
echo
"Install ${nginx_v} success"
echo
"Test http://$(ifconfig eth0 | awk -F"
[ :]+
" '/inet addr/{print $4}')"
else
echo
"Already installed ${nginx_v}"
fi
|
四、在两台Director上配置keepalived
1、配置keepalived
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
# vim /etc/keepalived/keepalived.conf
! Configuration File
for
keepalived
global_defs {
notification_email {
mail@huangming.org
741616710@qq.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_HA
}
vrrp_instance LVS_HA {
state BACKUP
# 主备服务器均为BACKUP
interface eth0
# 指定keepalived监听网络的接口
virtual_router_id 81
# ID标识,master和backup保持一致
priority 100
# backup为90
advert_int 2
# master和backup之间检查同步的时间间隔
nopreempt
# 不抢占模式,仅在master上设置
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.20
/24
dev eth0
# VIP,可以设置多个,每行一个
}
}
virtual_server 10.0.0.20 80 {
# (LVS配置)
delay_loop 5
# (每隔5秒查询realserver状态)
lb_algo wlc
# (lvs算法)
lb_kind DR
# (lvs负载均衡模式)
persistence_timeout 0
# (保持会话时间,0表示永久保持)
persistence_granularity 255.255.255.255
# (持久连接的粒度,默认为4个255,即一个单独的客户端IP分配到一个Realserver上)
protocol TCP
# (转发的协议类型,TCP或UDP)
sorry_server 10.0.0.11 8080
# (备用Realserver节点,当所有的realserver节点失效后,将启用这个节点)
real_server 10.0.0.13 80 {
# Realserver
weight 2
# (调度权重)
HTTP_GET {
# (健康检测方式)
url {
# (指定检测的url,可以指定多个)
path
/index
.html
# (url详细路径)
status_code 200
}
connect_port 80
# (检查的端口)
#bindto 10.0.0.20 # (表示通过此地址来发送请求对服务器进行健康检查)
connect_timeout 3
# (无响应超时时间)
nb_get_retry 3
# (失败重试次数)
delay_before_retry 1
# (重试时间间隔)
}
}
real_server 10.0.0.14 80 {
weight 2
# (调度权重)
HTTP_GET {
# (健康检测方式)
url {
# (指定检测的url,可以指定多个)
path
/index
.html
# (url详细路径)
status_code 200
}
connect_port 80
# (检查的端口)
#bindto 10.0.0.20 # (表示通过此地址来发送请求对服务器进行健康检查)
connect_timeout 3
# (无响应超时时间)
nb_get_retry 3
# (失败重试次数)
delay_before_retry 1
# (重试时间间隔)
}
}
|
2、分别启动keepalived服务
1
2
3
4
5
6
7
|
[root@node1 ~]
# service keepalived start
[root@node2 ~]
# service keepalived start
[root@node1 ~]
# ps aux | grep keepalived
root 4008 0.0 0.1 44928 1080 ? Ss 19:01 0:00 keepalived -D
root 4009 0.1 0.2 47152 2332 ? S 19:01 0:00 keepalived -D
root 4010 0.1 0.1 47032 1608 ? S 19:01 0:00 keepalived -D
root 4018 0.0 0.0 103256 864 pts
/0
S+ 19:03 0:00
grep
keepalived
|
3、查看LVS状态
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
|
# master上
[root@node1 ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.20:80 wlc persistent 0
-> 10.0.0.13:80 Route 2 0 0
-> 10.0.0.14:80 Route 2 0 0
# backup上
[root@node2 ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.20:80 wlc persistent 0
-> 10.0.0.13:80 Route 2 0 0
-> 10.0.0.14:80 Route 2 0 0
|
4、查看VIP(VIP默认分配在priority权值高的Director上)
1
2
3
4
5
6
7
8
9
10
11
12
|
[root@node1 ~]
# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link
/loopback
00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1
/8
scope host lo
inet6 ::1
/128
scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link
/ether
00:0c:29:5a:9d:e0 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.11
/24
brd 10.0.0.255 scope global eth0
inet 10.0.0.20
/24
scope global secondary eth0
inet6 fe80::20c:29ff:fe5a:9de0
/64
scope link
valid_lft forever preferred_lft forever
|
五、在两台Realserver上分别在回环接口上配置vip和关闭arp转发
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
# 查看系统默认arp规则
[root@node3 ~]
# cat /proc/sys/net/ipv4/conf/lo/arp_ignore
0
[root@node3 ~]
# cat /proc/sys/net/ipv4/conf/lo/arp_announce
0
[root@node3 ~]
# cat /proc/sys/net/ipv4/conf/all/arp_ignore
0
[root@node3 ~]
# cat /proc/sys/net/ipv4/conf/all/arp_announce
0
在两台Realserver上执行以下脚本
# lvs_dr_rs.sh
#!/bin/bash
vip=10.0.0.20
ifconfig
lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
echo
"1"
>
/proc/sys/net/ipv4/conf/lo/arp_ignore
echo
"2"
>
/proc/sys/net/ipv4/conf/lo/arp_announce
echo
"1"
>
/proc/sys/net/ipv4/conf/all/arp_ignore
echo
"2"
>
/proc/sys/net/ipv4/conf/all/arp_announce
[root@node3 ~]
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link
/loopback
00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1
/8
scope host lo
inet 10.0.0.20
/32
brd 10.0.0.20 scope global lo:0
inet6 ::1
/128
scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link
/ether
00:0c:29:c5:b4:19 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.13
/24
brd 10.0.0.255 scope global eth0
inet6 fe80::20c:29ff:fec5:b419
/64
scope link
valid_lft forever preferred_lft forever
[root@node4 ~]
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link
/loopback
00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1
/8
scope host lo
inet 10.0.0.20
/32
brd 10.0.0.20 scope global lo:0
inet6 ::1
/128
scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link
/ether
00:0c:29:42:34:4f brd ff:ff:ff:ff:ff:ff
inet 10.0.0.14
/24
brd 10.0.0.255 scope global eth0
inet6 fe80::20c:29ff:fe42:344f
/64
scope link
valid_lft forever preferred_lft forever
|
六、测试LVS负载功能
1
2
3
4
5
6
7
8
9
10
11
|
# 这里分别在node3和node4上访问VIP,在我的keepalived配置里我配置了wlc最小连接调度算法,
和权值为2,所以访问的效果如下
[root@node3 ~]
# curl http://10.0.0.20
10.0.0.13
[root@node3 ~]
# curl http://10.0.0.20
10.0.0.13
[root@node4 ~]
# curl http://10.0.0.20
10.0.0.14
[root@node4 ~]
# curl http://10.0.0.20
10.0.0.14
|
七、测试主Director故障,看是否实现故障转移(vip转至backup上)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
# 关闭主Director的keepalived服务
[root@node1 ~]
# service keepalived stop
Stopping keepalived: [ OK ]
# 查看BACKUP的ip
[root@node2 ~]
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN
link
/loopback
00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1
/8
scope host lo
inet6 ::1
/128
scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link
/ether
00:0c:29:42:43:20 brd ff:ff:ff:ff:ff:ff
inet 10.0.0.12
/24
brd 10.0.0.255 scope global eth0
inet 10.0.0.20
/24
scope global secondary eth0
inet6 fe80::20c:29ff:fe42:4320
/64
scope link
valid_lft forever preferred_lft forever
# 测试是否还能正常通过vip访问web服务
[root@node3 ~]
# curl http://10.0.0.20
10.0.0.13
[root@node3 ~]
# curl http://10.0.0.20
10.0.0.13
[root@node4 ~]
# curl http://10.0.0.20
10.0.0.14
[root@node4 ~]
# curl http://10.0.0.20
10.0.0.14
|
八、在Realserver配置DR模式启动服务脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
# cat lvs-dr-rs.service
#!/bin/bash
.
/etc/rc
.d
/init
.d
/functions
vip=10.0.0.20
RETVAL=$?
case
"$1"
in
start)
echo
"Start realserver"
ifconfig
lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev lo:0
echo
"1"
>
/proc/sys/net/ipv4/conf/lo/arp_ignore
echo
"2"
>
/proc/sys/net/ipv4/conf/lo/arp_announce
echo
"1"
>
/proc/sys/net/ipv4/conf/all/arp_ignore
echo
"2"
>
/proc/sys/net/ipv4/conf/all/arp_announce
;;
stop)
echo
"Stop realserver"
ifconfig
lo:0 down
echo
"0"
>
/proc/sys/net/ipv4/conf/lo/arp_ignore
echo
"0"
>
/proc/sys/net/ipv4/conf/lo/arp_announce
echo
"0"
>
/proc/sys/net/ipv4/conf/all/arp_ignore
echo
"0"
>
/proc/sys/net/ipv4/conf/all/arp_announce
;;
*)
echo
Usage: $0 {start|stop}
exit
1
esac
exit
$RETVAL
|
本文转自 HMLinux 51CTO博客,原文链接:http://blog.51cto.com/7424593/1827548