构建高可用nginx集群

一、案例例需求：

? 部署基于nginx分发器的高可用web集群

● 分发器故障自动切换

● 数据服务器自动容错

● 任何机器宕机不中断web业务

二、实验拓扑：

三、实验环境：

四、 实验步骤：

1.Master和Backup2个节点都需要安装nginx和keepalived

（其中nginx是以tar安装包方式进行编译安装，keepalived是用yum install keepalived -y安装）

无法使用killall 命令需要下载

yum -y install psmisc

master&backup节点都需要操作
#安装依赖包和keepalived
yum -y install gcc zlib zlib-devel pcre-devel openssl openssl-devel lsof elinks keepalived -y
#下载nginx安装包
wget http://nginx.org/download/nginx-1.15.5.tar.gz -P /usr/src
#安装包位置及解压
cd /usr/src
tar -zxvf nginx-1.15.5.tar.gz
#编译安装
cd nginx-1.15.5
./configure --prefix=/usr/local/nginx
make && make install
#杀死nginx
yum install psmisc
killall nginx
#启动nginx
/usr/local/nginx/sbin/nginx
#重启nginx
/usr/local/nginx/sbin/nginx -s reload
//检测nginx开启
lsof -i:80
COMMAND  PID   USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
nginx   4191   root    6u  IPv4  21646      0t0  TCP *:http (LISTEN)
nginx   4192 nobody    6u  IPv4  21646      0t0  TCP *:http (LISTEN)

2.master和backup 配置nginx和keepalived

01、master nginx配置

[root@vms40 ~]# cat /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
         upstream web {
                server 192.168.26.42 max_fails=2 fail_timeout=3;#3秒内失败2次，则认为此节点失效
                server 192.168.26.43 max_fails=2 fail_timeout=3;
        }
server {
        listen  80;
        server_name     localhost;
        location / {
        proxy_pass http://web;
        }
}
}
[root@vms40 ~]#systemctl enable nginx --now

02、master keepalived配置

[root@vms40 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
    router_id NGINX_DEVEL
}
vrrp_script check_nginx {
    script "/etc/keepalived/nginx_pid.sh"
    interval 2
    fall 1
}
vrrp_instance nginx {
    state MASTER
    interface ens32
    mcast_src_ip 192.168.26.40
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        check_nginx
    }
    virtual_ipaddress {
        192.168.26.150/24
    }
}
[root@vms40 keepalived]# ip addr
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:45:e5:48 brd ff:ff:ff:ff:ff:ff
    inet 192.168.26.40/24 brd 192.168.26.255 scope global ens32
       valid_lft forever preferred_lft forever
    inet 192.168.26.150/24 scope global secondary ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe45:e548/64 scope link 
       valid_lft forever preferred_lft forever
[root@vms40 keepalived]# 

02-1、构建关联脚本

[root@vms40 ~]# cat /etc/keepalived/nginx_pid.sh 
#!/bin/bash
nginx_kp_check () {
    nginxpid=`ps -C nginx --no-header |wc -l`
    if [ $nginxpid -eq 0 ];then
        /usr/local/nginx/sbin/nginx
        sleep 1
        nginxpid=`ps -C nginx --no-header |wc -l`
        if [ $nginxpid -eq 0 ];then
            systemctl stop keepalived
        fi
    fi
}

nginx_kp_check
[root@vms40 ~]#chmod 755 /etc/keepalived/nginx_pid.sh
[root@vms40 ~]#systemctl enable keepalived --now   #开机和现在启动

脚本功能说明：统计nginx进程数量，如果进程数量的值等于0，说明nginx挂了，那么执行/usr/local/nginx/sbin/nginx去启动分发器，等待1秒后再次检查进程数量，如果进程数量的值还是等于0，则执行systemctl stop keepalived停止keepalived服务。这样就停止发组播，释放VIP，而备用服务器就开始接手工作了。

03、Backup nginx配置

[root@vms41 ~]# cat /usr/local/nginx/conf/nginx.conf
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
         upstream web {
                server 192.168.26.42 max_fails=2 fail_timeout=3;#3秒内失败2次，则认为此节点失效
                server 192.168.26.43 max_fails=2 fail_timeout=3;
        }
server {
        listen  80;
        server_name     localhost;
        location / {
        proxy_pass http://web;
        }
}
}
[root@vms41 ~]# systemctl enable nginx --now   #开机和现在启动

04、Backup keepalived配置

[root@vms41 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
global_defs {
    router_id NGINX_DEVEL
}
vrrp_script check_nginx {
    script "/etc/keepalived/nginx_pid.sh"
    interval 2
    fall 1
}
vrrp_instance nginx {
    state BACKUP
    interface ens32
    mcast_src_ip 192.168.26.41
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    track_script {
        check_nginx
    }
    virtual_ipaddress {
        192.168.26.150/24
    }
}
[root@vms41 ~]# ip add      
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:27:1c:a2 brd ff:ff:ff:ff:ff:ff
    inet 192.168.26.41/24 brd 192.168.26.255 scope global ens32
       valid_lft forever preferred_lft forever
    inet 192.168.26.150/24 scope global secondary ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe27:1ca2/64 scope link 
       valid_lft forever preferred_lft forever
[root@vms41 ~]# 

04-1、构建关联脚本

[root@vms41 ~]# cat /etc/keepalived/nginx_pid.sh 
#!/bin/bash
nginx_kp_check () {

#检测nginx服务
nginxpid=`ps -C nginx --no-header |wc -l`
if [ $nginxpid -eq 0 ];then

#检测不到时启动nginx
/usr/local/nginx/sbin/nginx

#1秒后继续检测
sleep 1
nginxpid=`ps -C nginx --no-header |wc -l`
if [ $nginxpid -eq 0 ];then

#启动服务失败后关闭keepalived
systemctl stop keepalived
fi
fi
}
nginx_kp_check
[root@vms41 ~]#chmod 755 /etc/keepalived/nginx_pid.sh
[root@vms41 ~]#systemctl enable keepalived --now   #开机和现在启动

05.web1、web2节点分别安装nginx和配置

#web1、web2下载nginx
yum install nginx -y
#web1、web2配置
[root@vms42 ~]# echo "web1" > /usr/share/nginx/html/index.html 
[root@vms42 ~]# curl 192.168.26.42
web1
[root@vms43 ~]# echo "web2" > /usr/share/nginx/html/index.html 
[root@vms43 ~]# curl 192.168.26.43
web2

五、 集群高可用性测试

集群测试：使用客户端正常访问VIP

访问keepalived配置的公网vip

停止nginx服务，nginx_pid.sh检测脚本将自动启动nginx

#杀死nginx服务[root@localhost ~]# killall nginx
#再次查看nginx_pid.sh是否生效，检测不到时启动nginx，已经杀死了nginx,脚本再次启动nginx,查看80端口是否启动。
[root@localhost ~]# lsof -i:80

COMMAND   PID   USER   FD   TYPE DEVICE SIZE/OFF NODE NAME

nginx   18462   root    6u  IPv4  59924      0t0  TCP *:http (LISTEN)

nginx   18464 nobody    6u  IPv4  59924      0t0  TCP *:http (LISTEN)

强制停止master的nginx服务，主检测脚本发现不能启动nginx，直接杀死主keepalived服务，backup自己组播vip代替master

[root@vms40 ~]# watch -n1 'killall nginx'
每隔秒执行一次killall nginx命令
[root@40 ~]# ip add #查看VIP是否还在
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:45:e5:48 brd ff:ff:ff:ff:ff:ff
    inet 192.168.26.40/24 brd 192.168.26.255 scope global ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe45:e548/64 scope link 
       valid_lft forever preferred_lft forever

恢复master的keepalived服务，正常运行，backup退出vip

[root@vms40 keepalived]# systemctl  restart keepalived.service 
[root@vms40 keepalived]# ip addr
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens32:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:45:e5:48 brd ff:ff:ff:ff:ff:ff
    inet 192.168.26.40/24 brd 192.168.26.255 scope global ens32
       valid_lft forever preferred_lft forever
    inet 192.168.26.150/24 scope global secondary ens32
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe45:e548/64 scope link 
       valid_lft forever preferred_lft forever
[root@vms40 keepalived]# lsof -i:80
COMMAND   PID   USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
nginx   24068   root    6u  IPv4  71692      0t0  TCP *:http (LISTEN)
nginx   24070 nobody    6u  IPv4  71692      0t0  TCP *:http (LISTEN)
[root@vms40 keepalived]# 

web服务器故障测试

web服务器容错机制

每个设备的状态设置为:
1.down 表示单前的server暂时不参与负载
2.weight 默认为1.weight越?，负载的权重就越?。
3.max_fails ：允许请求失败的次数默认为1.当超过最?次数时，返回
proxy_next_upstream 模块定义的错误
4.fail_timeout:失败超时时间，在连接Server时，如果在超时时间之内超过
max_fails指定的失败次数，会认为在fail_timeout时间内Server不可?。默认为
10s。
5.backup： 其它所有的?backup机器down或者忙的时候，请求backup机器。所以这台
机器压?会最轻。

在nginx配置文件中配置rs

upstream web{

        server 192.168.42.134 max_fails=2 fail_timeout=5;

        server 192.168.42.135 max_fails=2 fail_timeout=5;

web01服务器故障，分发器自动将请求分发到web02上面，web01 nginx恢复，请求分发正常

[root@vms42 ~]# killall nginx
[root@vms42 ~]# lsof -i:80
[root@vms42 ~]# curl  192.168.26.150
web2
[root@vms42 ~]# curl  192.168.26.150
web2
[root@vms42 ~]# curl  192.168.26.150
web2
[root@vms42 ~]# curl  192.168.26.150
web2
#重启web1 nginx
[root@vms42 ~]# systemctl  restart nginx
[root@vms42 ~]# curl  192.168.26.150
web2
[root@vms42 ~]# curl  192.168.26.150
web1
[root@vms42 ~]# curl  192.168.26.150
web2
[root@vms42 ~]# curl  192.168.26.150
web1