Nginx+keepalived做双机热备加tomcat负载均衡

环境说明：

nginx1:
192.168.2.47
nginx2:
192.168.2.48
tomcat1:
192.168.2.49
tomcat2:
192.168.2.50
vip:
192.168.2.51

一.Nginx配置
1.安装Nginx所需pcre库
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.10.tar.gz

tar -
zxvf pcre-
8.10
.tar
.gz

cd pcre-
8.10

./
configure
make
make install

2.安装Nginx
wget http://nginx.org/download/nginx-0.8.52.tar.gz

groupadd www
useradd -
g www www
tar zxvf nginx-
0.8.52.tar
.gz

cd nginx-
0.8.52/

./
configure --
user=
www --
group=
www --
prefix=/
usr/
local/
nginx --
with-
http_stub_status_module --
with-
http_ssl_module
make
make install

注：如果出现以下错误

./
configure:
 error:
 SSL modules require the OpenSSL library.
Centos
需要安装openssl-
devel
Ubuntu则需要安装:
sudo apt-
get install libssl-
dev

3.修改配置文件为以下内容：

user  www www;

worker_processes 2
;

pid        logs/
nginx.pid
;

worker_rlimit_nofile 51200
;

 
events
{

 use epoll;

 worker_connections 51200
;

}

 
http
{

 include       mime.types
;

 default_type  application/
octet-
stream;

 keepalive_timeout 120
;

 server_tokens off;

 send_timeout 60
;

 tcp_nodelay on;

 
 upstream  tomcats  {

 server 192.168.2.50:
8080
;

 server 192.168.2.49:
8080
;

 #ip_hash;
       #在没有做共享session的情况下ip_hash可以解决session问题
 
 }

 
 server
 {

 listen  80
;

 server_name  192.168.2.48;

 
 location /
 {

 proxy_pass        http:
//tomcats;

 proxy_set_header   Host             $host;

 proxy_set_header   X-
Real-
IP        $remote_addr;

 proxy_set_header   X-
Forwarded-
For
  $proxy_add_x_forwarded_for;

 }

 
 log_format access_log  '$remote_addr - $remote_user [$time_local] $request '

 '"$status" $body_bytes_sent "$http_referer" '

 '"$http_user_agent" "$http_x_forwarded_for"'
;

 access_log  /
usr/
local/
nginx/
logs/
access.log
  access_log;

 }

 
}

4.测试配置文件

/
usr/
local/
nginx/
sbin/
nginx -
t

如果出现以下情况

/
usr/
local/
nginx/
sbin/
nginx:
 error while
 loading shared libraries:
 libpcre.so
.0:
 or directory

解决方法：

sudo ln -
s /
usr/
local/
lib/
libpcre.so
.0 /
usr/
lib/
libpcre.so
.0

/usr/local/nginx/sbin/nginx -t
显示以下信息为正确的

the configuration file /
usr/
local/
nginx/
conf/
nginx.conf
 syntax is ok
configuration file /
usr/
local/
nginx/
conf/
nginx.conf
 test is successful

5.优化内核参数
vim /etc/sysctl.conf在最后添加

net.ipv4
.tcp_max_syn_backlog
 =
 65536

net.core
.netdev_max_backlog
 =
 32768

net.core
.somaxconn
 =
 32768

net.core
.wmem_default
 =
 8388608

net.core
.rmem_default
 =
 8388608

net.core
.rmem_max
 =
 16777216

net.core
.wmem_max
 =
 16777216

net.ipv4
.tcp_timestamps
 =
 0

net.ipv4
.tcp_synack_retries
 =
 2

net.ipv4
.tcp_syn_retries
 =
 2

net.ipv4
.tcp_tw_recycle
 =
 1

net.ipv4
.tcp_tw_reuse
 =
 1

net.ipv4
.tcp_mem
 =
 94500000
 915000000
 927000000

net.ipv4
.tcp_max_orphans
 =
 3276800

net.ipv4
.ip_local_port_range
 =
 1024
  65535

保存退出后执行

sysctl -
p

6.切割Nginx日志脚本

#!/
bin/
bash
PATH_LOGS=
"/usr/local/nginx/logs"

YEAR=
`date -
d "-1 days"
 +
"%Y"
`
MONTH=
`date -
d "-1 days"
 +
"%m"
`
mkdir -
p $PATH_LOGS/
$YEAR/
$MONTH
mv $PATH_LOGS/
access.log
 $PATH_LOGS/
$YEAR/
$MONTH/
access_$(
date -
d "-1 days"
 +
"%Y%m%d"
)
.log

kill -
USR1 `cat $PATH_LOGS/
nginx.pid
`

把该脚本加到crontab每天00点执行
注：备机的Nginx和以上安装步骤一样

二.安装配置Keepalived
1.下载所需要的软件
wget http://keepalived.org/software/keepalived-1.1.19.tar.gz
wget http://rpm5.org/files/popt/popt-1.16.tar.gz
2.安装popt
编译keepalived时需要popt，否则会报以下错误：

configure:
 error:
 Popt libraries is required

tar -
zxvf popt-
1.16
.tar
.gz

cd popt-
1.16

./
configure
make
make install

3.安装keepalived

tar -
zxvf keepalived-
1.1.19.tar
.gz

cd keepalived-
1.1.19
./
configure --
prefix=/
usr/
local/
keepalived
make
make install

4.修改配置文件为以下内容:

vim /usr/local/keepalived/etc/keepalived/keepalived.conf

!
 Configuration File
 for
 keepalived
 
global_defs {

 router_id LVS_DEVEL
}

vrrp_script Monitor_Nginx {

 script "/root/scripts/monitor_nginx.sh"
 #根据自己的实际路径放置monitor_nginx.sh
    
 interval 2

 weight 2

}

vrrp_instance VI_1 {

 state MASTER
 interface
 eth0
 virtual_router_id 51

 priority 100

 advert_int 1

 authentication {

 auth_type PASS
 auth_pass 1234

}

 track_script {

 Monitor_Nginx
}

 virtual_ipaddress {

 192.168.2.51
 }

}

注：monitor_nginx.sh为监控nginx进程的脚本,内容如下

#!/
bin/
bash
if
 [
 "$(ps -ef | grep "
nginx:
 master process"| grep -v grep )"
 ==
 ""
 ]

then
 /
usr/
local/
nginx/
sbin/
nginx
 sleep 5

 if
 [
 "$(ps -ef | grep "
nginx:
 master process"| grep -v grep )"
 ==
 ""
 ]

 then
 killall keepalived
 fi
fi

5.启动keepalived

/
usr/
local/
keepalived/
sbin/
keepalived -
D -
f /
usr/
local/
keepalived/
etc/
keepalived/
keepalived.conf

注:备机的keepalived的安装和上面一样，只要把配置文件改为以下(把MASTER改为BACKUP)

!
 Configuration File
 for
 keepalived
 
global_defs {

 router_id LVS_DEVEL
}

vrrp_script Monitor_Nginx {

 script "/root/scripts/monitor_nginx.sh"

 interval 2

 weight 2

}

vrrp_instance VI_1 {

 state BACKUP            #改为BACKUP
 interface
 eth0
 virtual_router_id 51

 priority 100
            #比MASTER数值要低
 advert_int 1

 authentication {

 auth_type PASS
 auth_pass 1234

}

 track_script {

 Monitor_Nginx
}

 virtual_ipaddress {

 192.168.2.51
 }

}

三．测试步骤

1．    访问VIP看是否能够正常访问后端的tomcat
2．    停止其中一个tomcat看是否能将访问转到另一台上
3．    停止两台nginx上任何一个nginx进程看监控进程脚本是否会自动启动nginx
4．    停止任何一台nginx上的keepalived进程看另一台是否接管vip
比如停止Master上的keepalived,例如如下killall keepalived,查看BACKUP机器是否已经接管，如果BACKUP接管后，BACKUP机器日志会是出下情况
tail  /var/log/syslog

Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Transition to MASTER STATE
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Entering MASTER STATE
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 setting protocol VIPs.
Keepalived_vrrp
:
 VRRP_Instance(
VI_1)
 Sending gratuitous ARPs on eth0 for
 192.168.2.51

MASTER机器上日志会显示

Keepalived_vrrp:
 Terminating VRRP child process on signal
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 removing protocol VIPs.

现在把MASTER上的Keepalived重新启动，会看到MASTER重新接管VIP，并对外提供服务，BACKUP仍旧回到BACKUP STATE,如果不是这种情况，请检查配置文件和步骤.

现在的BACKUP日志如下：

Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Received higher prio advert
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Entering BACKUP STATE
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 removing protocol VIPs.

Master日志如下：

Keepalived_vrrp:
 VRRP_Script(
Monitor_Nginx)
 succeeded
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Transition to MASTER STATE
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 Entering MASTER STATE
Keepalived_vrrp:
 VRRP_Instance(
VI_1)
 setting protocol VIPs.
Keepalived_vrrp
:
 VRRP_Instance(
VI_1)
 Sending gratuitous ARPs on eth0 for
 192.168.2.51