神出鬼没的Enderman

负载均衡企业实践应用

根据用户访问的uri信息进行负载均衡

1. 提高网站服务安全性
2. 管理操作工作简化
3. 可以换分不同人员管理不同集群服务器
4. 实现网站集群动静分离

• 负载均衡服务器配置

www.1997sty.com域名按照uri解析到不同的服务器上

upstream upload {
   server 10.0.0.8:80;
}
upstream static {
   server 10.0.0.7:80;
}
upstream default {
   server 10.0.0.9:80;
}
 
 
server {
    listen       80;
    server_name  www.1997sty.com;
    location / {
       proxy_pass http://default;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
    location /upload {
       proxy_pass http://upload;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
    location /static {
       proxy_pass http://static;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
}

虽然3台服务器都设置了相同的站点目录,但只会访问某个服务器的其中一个分支目录

• web服务器10.0.0.7

mkdir -p /html/www{,/upload/static}
echo "static_10.0.0.7" >/html/www/static/index.html
echo "upload_10.0.0.7" >/html/www/upload/index.html
echo "default_10.0.0.7" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• web服务器10.0.0.8

mkdir -p /html/www{,/upload/static}
echo "static_10.0.0.8" >/html/www/static/index.html
echo "upload_10.0.0.8" >/html/www/upload/index.html
echo "default_10.0.0.8" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• web服务器10.0.0.7

mkdir -p /html/www{,/upload/static}
echo "static_10.0.0.9" >/html/www/static/index.html
echo "upload_10.0.0.9" >/html/www/upload/index.html
echo "default_10.0.0.9" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• 修改配置后重启nginx,访问效果

根据用户访问的终端信息显示不同页面

• 负载均衡服务器配置

www.1997sty.com域名按照user-agent区分访问的客户端类型

upstream web {
   server 10.0.0.8:80;
}
upstream mobile {
   server 10.0.0.7:80;
}
upstream default {
   server 10.0.0.9:80;
}
 
 
server {
    listen       80;
    server_name  www.1997sty.com;
    location / {
       if ($http_user_agent ~* iphone) {
          proxy_pass http://mobile;
       }
       if ($http_user_agent ~* Chrome) {
         proxy_pass  http://web;
       }
       proxy_pass http://default;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
}

• web服务器10.0.0.7

mkdir -p /html/www
echo "iphone_access 10.0.0.7" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• web服务器10.0.0.8

mkdir -p /html/www
echo "google_access 10.0.0.8" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• web服务器10.0.0.9

mkdir -p /html/www
echo "default_access 10.0.0.9" >/html/www/index.html
 
vi /etc/nginx/www.conf
 
server {
    listen       80;
    server_name  www.1997sty.com;
 
    location / {
        root   /html/www;
        index  index.html index.htm;
    }
}

• 修改配置后重启nginx,访问效果

高可用keepalived服务

主机名称和IP地址规划

服务	主机名	eth0网卡IP	eth1网卡IP	软件
防火墙服务器	firewalld	10.0.0.81	172.16.1.81	firewalld
负载均衡服务器	lb01	10.0.0.5	172.16.1.5	nginx,keepalived
负载均衡服务器	lb02	10.0.0.6	172.16.1.6	nginx,keepalived
web服务器	web01	10.0.0.7	172.16.1.7	nginx
web服务器	web02	10.0.0.8	172.16.1.8	nginx
web服务器	web03	10.0.0.9	172.16.1.9	nginx
存储服务器	nfs01	10.0.0.31	172.16.1.31	nfs
备份服务器	backup	10.0.0.41	172.16.1.41	rsync
数据库服务器	db01	10.0.0.51	172.16.1.51	mysql,mariaDB
批量管理服务器	m01	10.0.0.61	172.16.1.61	ansible
跳板机服务器	jumpserver	10.0.0.71	172.16.1.71	jumpserver
监控服务器	zabbix	10.0.0.72	172.16.1.72	zabbix
缓存服务器	暂无	暂无	暂无	暂无

避免负载均衡服务出现单点问题

高可用服务的原理

高可用keepalived服务部署流程

安装部署keepalived软件

yum install -y keepalived

编写keepalived配置文件

• /etc/keepalived/keepalived.conf : keepalived配置文件
• GLOBAL CONFIGURATION : 全局配置部分
• VRRPD CONFIGURATION : VRRP协议配置部分
• LVS CONFIGURATION : LVS服务管理配置部分

! Configuration File for keepalived
#全局配置部分
global_defs {
   #设置发送邮件信息的收件人
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   #设置连接的邮件服务器信息
   notification_email_from 1997sty@163.com
   smtp_server 163.smtp.xxx_
   smtp_connect_timeout 30
   #高可用集群主机身份标识(集群中主机身份标识名称不能重复)
   router_id LVS_DEVEL
}
#Vrrp协议家族
vrrp_instance 1997sty {
    #标识所在家族中的身份 (MASTER/BACKUP)
    state MASTER
    #指定虚拟IP地址出现在什么网卡上
    interface eth0
    #标识家族身份信息 多台高可用服务配置要一致
    virtual_router_id 51
    #设定优先级 优先级越高,就越有可能成为主
    priority 100
    #定义组播包发送的间隔时间(秒)  主和备配置一样
    advert_int 1
    #实现通讯需要有认证过程
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    #配置虚拟IP地址信息
    virtual_ipaddress {
        192.168.200.16
        192.168.200.17
        192.168.200.18
    }
}

• 修改10.0.0.5的/etc/keepalived/keepalived.conf配置文件

! Configuration File for keepalived
 
global_defs {
    router_id lb01
}
 
vrrp_instance 1997sty {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.3/24
    }
}

• 修改10.0.0.6的/etc/keepalived/keepalived.conf配置文件

! Configuration File for keepalived
 
global_defs {
   router_id lb02
}
 
vrrp_instance 1997sty {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.3/24
    }   
}

配置完成后启动keepalived服务

systemctl start keepalived.service
systemctl status keepalived.service
#查看ip信息,MASTER主机的ip会增加一个keepalived的ip地址
ip a

• 分别访问主备后抓包记录

高可用服务脑裂问题

原因是高可用备服务器接收不到主服务器发送的组播包,备服务器上会自动生成VIP地址

• 脑裂前抓包

• 脑裂后抓包

产生原因

• 物理原因: 高可用集群之间通讯线路出现问题
• 逻辑原因: 有安全策略阻止

解决方法

进行监控,发出邮件警告

• 在10.0.0.6上部署该脚本,主机不可用,备份机立刻启动并发送邮件

#!/bin/bash
#keepalived备份机监控,当备份机运行时执行该脚本就会发送邮件
ip a s eth0|grep "10.0.0.3" >/dev/null
if [ $? -eq 0 ]
then
  echo "keepalived服务出现异常,请进行检查"|mail -s 异常告警-keepalived  admin@1997sty.com
fi
 
# shell脚本进行比较判断
# -eq  等于
# -ne  不等于
# -lt  小于
# -gt  大于
# -le  小于等于
# -ge  大于等于

• 使用邮件服务需要配置/etc/mail.rc,并重启邮件服务systemctl restart postfix.service

vim /etc/mail.rc
 
set from=邮箱地址 smtp=邮件服务器地址
set smtp-auth-user=邮箱地址 smtp-auth-password=邮箱授权密码 smtp-auth=login
 
systemctl restart postfix.service

关闭其中一台服务器keepalived服务

• 在10.0.0.5编写监控nginx服务状态监控,作用是监控nginx是否启动,如果nginx不可用,这台服务器的keepalived也要关闭

#!/bin/bash
#不写成ps -ef|grep -c nginx主要是筛除了grep --color=auto nginx这条记录
#nginx服务停止时,同时停止keepalived服务
num=`ps -ef|grep -c [n]ginx`
if [ $num -lt 2 ]
then
   systemctl stop keepalived
fi

• 修改10.0.0.5的/etc/keepalived/keepalived.conf配置文件

! Configuration File for keepalived
global_defs {
    router_id lb01
}
 
vrrp_script check_web {
    #定义需要监控脚本(脚本有执行权限)
    script "/root/check_web.sh"
    #执行脚本的间隔时间(秒)
    interval 3
    weight 2
}
 
vrrp_instance 1997sty {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.3/24
    }
    track_script {
        #调用执行你的脚本信息
        check_web
   }
}

高可用集群双主配置

修改keepalived配置文件

• 修改10.0.0.5的/etc/keepalived/keepalived.conf配置文件

! Configuration File for keepalived
global_defs {
   router_id lb01
}
 
vrrp_instance 1997sty {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
     10.0.0.3/24
    }
}
vrrp_instance sty1997 {
    state BACKUP
    interface eth0
    virtual_router_id 52
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
     10.0.0.4/24
    }
}

• 修改10.0.0.6的/etc/keepalived/keepalived.conf配置文件

! Configuration File for keepalived
global_defs {
   router_id lb02
}
 
vrrp_instance 1997sty {
    state BACKUP
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.3/24
    }
}
vrrp_instance sty1997 {
    state MASTER
    interface eth0
    virtual_router_id 52
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.4/24
    }
}

重启keepalived服务后查看网卡信息

如果其中任意一台失效,另一台主机就会同时拥有10.0.0.3和10.0.0.4的地址

• 10.0.0.5

[root@lb01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:20:f8:30 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.5/24 brd 10.0.0.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.0.0.3/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::e07d:4f53:a2a8:f415/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::40b1:ca4:d21:8f0b/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::2bf8:4ec5:b3b1:5c3b/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:20:f8:3a brd ff:ff:ff:ff:ff:ff
    inet 172.16.1.5/24 brd 172.16.1.255 scope global noprefixroute eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::8b98:49b4:5566:ec9d/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

• 10.0.0.6

[root@lb02 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:56:e6:44 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.6/24 brd 10.0.0.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 10.0.0.4/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::e07d:4f53:a2a8:f415/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::40b1:ca4:d21:8f0b/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::2bf8:4ec5:b3b1:5c3b/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:56:e6:4e brd ff:ff:ff:ff:ff:ff
    inet 172.16.1.6/24 brd 172.16.1.255 scope global noprefixroute eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::5ef1:fc45:5747:6395/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
    inet6 fe80::8b98:49b4:5566:ec9d/64 scope link tentative noprefixroute dadfailed 
       valid_lft forever preferred_lft forever

高可用服务安全访问配置(负载均衡服务)

• 配置10.0.0.5和10.0.0.6的/etc/nginx/conf.d/www.conf配置文件

upstream 1997sty {
   server 10.0.0.7:80;
   server 10.0.0.8:80;
   server 10.0.0.9:80;
}
server {
    listen       10.0.0.3:80;
    server_name  www.1997sty.com;
    location / {
       proxy_pass http://1997sty;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
}
server {
    listen       10.0.0.4:80;
    server_name  bbs.1997sty.com;
    location / {
       proxy_pass http://1997sty;
       proxy_set_header Host $host;
       proxy_set_header X-Forwarded-For $remote_addr;
       proxy_next_upstream error timeout http_404 http_502 http_403;
    }
}

如果直接重启服务不会成功,默认nginx不能监听网卡上没有的地址

• 解决方法: 修改内核信息

echo 'net.ipv4.ip_nonlocal_bind = 1' >>/etc/sysctl.conf
sysctl -p
systemctl restart nginx
#如果使用reload重启服务,不会改变监听地址的信息,必须使用restart
#systemctl reload nginx
#使用命令查看端口,如果监听为全局地址,可以移除nginx其他的配置文件
netstat -luntp |grep 80

• nginx监听结果

[root@lb01 ~]# netstat -luntp |grep 80
tcp        0      0 10.0.0.4:80             0.0.0.0:*               LISTEN      4127/nginx: master  
tcp        0      0 10.0.0.3:80             0.0.0.0:*               LISTEN      4127/nginx: master 
[root@lb02 ~]# netstat -luntp |grep 80
tcp        0      0 10.0.0.4:80             0.0.0.0:*               LISTEN      3529/nginx: master  
tcp        0      0 10.0.0.3:80             0.0.0.0:*               LISTEN      3529/nginx: master