//先具备单master节点部署环境 接K8S单节点部署
master01:192.168.20.10 kube-apiserver kube-controller-manager kube-scheduler etcd master02:192.168.20.40 kube-apiserver kube-controller-manager kube-scheduler node1:192.168.20.20 kubelet kube-proxy docker flannel etcd node2:192.168.20.30 kubelet kube-proxy docker flannel etcd
master02部署
[root@localhost ~]# hostnamectl set-hostname master2 [root@localhost ~]# su//优先关闭防火墙和selinux服务
[root@master2 ~]# iptables -F [root@master2 ~]# setenforce 0//在master01上操作 //复制kubernetes目录到master02
[root@master ~]# scp -r /opt/kubernetes/ root@192.168.20.40:/opt The authenticity of host '192.168.20.40 (192.168.20.40)' can't be established. ECDSA key fingerprint is SHA256:v7t4p3JJLUnXziTqE64SOtmKTkJdbSB2hEykd+xG22c. ECDSA key fingerprint is MD5:85:90:0a:05:38:e9:e3:37:25:de:f0:08:71:9e:9d:c5. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added '192.168.20.40' (ECDSA) to the list of known hosts. root@192.168.20.40's password: token.csv 100% 84 14.3KB/s 00:00 kube-apiserver 100% 929 96.4KB/s 00:00 kube-scheduler 100% 94 21.3KB/s 00:00 kube-controller-manager 100% 483 82.4KB/s 00:00 kube-apiserver 100% 184MB 20.4MB/s 00:09 kubectl 100% 55MB 13.7MB/s 00:04 kube-controller-manager 100% 155MB 16.9MB/s 00:09 kube-scheduler 100% 55MB 18.6MB/s 00:02 ca-key.pem 100% 1679 341.9KB/s 00:00 ca.pem 100% 1359 192.7KB/s 00:00 server-key.pem 100% 1679 301.5KB/s 00:00 server.pem 100% 1643 224.1KB/s 00:00//复制master中的三个组件启动脚本kube-apiserver.service kube-controller-
manager.service kube-scheduler.service [root@master ~]# scp /usr/lib/systemd/system/{kube-apiserver,kube-controller-manager,kube-scheduler}.service root@192.168.20.40:/usr/lib/systemd/system/ root@192.168.20.40's password: kube-apiserver.service 100% 282 72.7KB/s 00:00 kube-controller-manager.service 100% 317 55.7KB/s 00:00 kube-scheduler.service 100% 281 42.4KB/s 00:00//master02上操作 //修改配置文件kube-apiserver中的IP
[root@master2 ~]# cd /opt/kubernetes/cfg/ [root@master2 cfg]# vim kube-apiserver KUBE_APISERVER_OPTS="--logtostderr=true \ --v=4 \ --etcd-servers=https://192.168.20.10:2379,https://192.168.20.20:2379,https://192.168.20.30:2379 \ --bind-address=192.168.20.40 \ //这里改成master2的地址 --secure-port=6443 \ --advertise-address=192.168.20.40 \ //这里改成master2的地址 --allow-privileged=true \ --service-cluster-ip-range=10.0.0.0/24 \ --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \ --authorization-mode=RBAC,Node \ --kubelet-https=true \ --enable-bootstrap-token-auth \ --token-auth-file=/opt/kubernetes/cfg/token.csv \ --service-node-port-range=30000-50000 \ --tls-cert-file=/opt/kubernetes/ssl/server.pem \ --tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \ --client-ca-file=/opt/kubernetes/ssl/ca.pem \ --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \ --etcd-cafile=/opt/etcd/ssl/ca.pem \ --etcd-certfile=/opt/etcd/ssl/server.pem \ --etcd-keyfile=/opt/etcd/ssl/server-key.pem"//特别注意：master02一定要有etcd证书 //需要拷贝master01上已有的etcd证书给master02使用 //在master01上操作
[root@master ~]# scp -r /opt/etcd/ root@192.168.20.40:/opt/ root@192.168.20.40's password: etcd 100% 509 57.4KB/s 00:00 etcd 100% 18MB 19.9MB/s 00:00 etcdctl 100% 15MB 15.1MB/s 00:01 ca-key.pem 100% 1675 144.8KB/s 00:00 ca.pem 100% 1265 371.3KB/s 00:00 server-key.pem 100% 1679 287.1KB/s 00:00 server.pem 100% 1338 439.4KB/s 00:00//在master02上操作 //启动master02中的三个组件服务
[root@master2 cfg]# systemctl start kube-apiserver.service [root@master2 cfg]# systemctl start kube-controller-manager.service [root@master2 cfg]# systemctl start kube-scheduler.service//增加环境变量
[root@master2 cfg]# vim /etc/profile #末尾添加 export PATH=$PATH:/opt/kubernetes/bin/ [root@master2 cfg]# source /etc/profile [root@master2 cfg]# kubectl get node NAME STATUS ROLES AGE VERSION 192.168.20.20 Ready <none> 7d22h v1.12.3 192.168.20.30 Ready <none> 7d21h v1.12.3//先具备多master节点部署环境 接K8S多节点部署 master1:192.168.20.10 kube-apiserver kube-controller-manager kube-scheduler etcd master2:192.168.20.40 node1:192.168.20.20 kubelet kube-proxy docker flannel etcd node2:192.168.20.30 kubelet kube-proxy docker flannel etcd 漂移地址:192.168.20.111 负载均衡LoadBalance lb01:master 192.168.20.50 lb02:backup 192.168.20.60
//lb01 lb02操作 192.168.20.50
[root@localhost ~]# hostnamectl set-hostname lb01 [root@localhost ~]# su192.168.20.60
[root@localhost ~]# hostnamectl set-hostname lb02 [root@localhost ~]# su//优先关闭lb01和lb02的防火墙和selinux服务
systemctl stop firewalld.service setenforce 0//安装nginx服务，把nginx.sh和keepalived.conf脚本拷贝到家目录
vim /etc/yum.repos.d/nginx.repo [nginx] name=nginx repo baseurl=http://nginx.org/packages/centos/7/$basearch/ gpgcheck=0 yum install nginx -y //添加四层转发（在events {}http {}中间添加） vim /etc/nginx/nginx.conf events { worker_connections 1024; } stream { log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; access_log /var/log/nginx/k8s-access.log main; upstream k8s-apiserver { server 192.168.20.10:6443; //master01地址 server 192.168.20.40:6443; //master02地址 } server { listen 6443; proxy_pass k8s-apiserver; } } http {//启动nginx服务
systemctl start nginx//部署keepalived服务
yum install keepalived -y//修改配置文件
cp keepalived.conf /etc/keepalived/keepalived.conf cp：是否覆盖"/etc/keepalived/keepalived.conf"？ yes//注意：lb01是Mster配置如下：
[root@lb01 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { # 接收邮件地址 notification_email { acassen@firewall.loc failover@firewall.loc sysadmin@firewall.loc } # 邮件发送地址 notification_email_from Alexandre.Cassen@firewall.loc smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGINX_MASTER } vrrp_script check_nginx { script "/etc/nginx/check_nginx.sh" } vrrp_instance VI_1 { state MASTER interface ens33 virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 priority 100 # 优先级，备服务器设置 90 advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 192.168.20.111/24 } track_script { check_nginx } }//注意：lb02是Backup配置如下：
[root@lb02 ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { # 接收邮件地址 notification_email { acassen@firewall.loc failover@firewall.loc sysadmin@firewall.loc } # 邮件发送地址 notification_email_from Alexandre.Cassen@firewall.loc smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGINX_MASTER } vrrp_script check_nginx { script "/etc/nginx/check_nginx.sh" } vrrp_instance VI_1 { state BACKUP interface ens33 virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 priority 90 # 优先级，备服务器设置 90 advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 192.168.20.111/24 } track_script { check_nginx } } vim /etc/nginx/check_nginx.sh count=$(ps -ef |grep nginx |egrep -cv "grep|$$") if [ "$count" -eq 0 ];then systemctl stop keepalived fi chmod +x /etc/nginx/check_nginx.sh systemctl start keepalived//查看lb01地址信息
[root@lb01 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:7f:10:c4 brd ff:ff:ff:ff:ff:ff inet 192.168.20.50/24 brd 192.168.20.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet 192.168.20.111/24 scope global secondary ens33 valid_lft forever preferred_lft forever inet6 fe80::2699:92a4:f2b8:7f88/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000 link/ether 52:54:00:64:d5:ef brd ff:ff:ff:ff:ff:ff inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 valid_lft forever preferred_lft forever 4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000 link/ether 52:54:00:64:d5:ef brd ff:ff:ff:ff:ff:ff//漂移地址在lb01中
//查看lb02地址信息
[root@lb02 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:6e:8e:cd brd ff:ff:ff:ff:ff:ff inet 192.168.20.60/24 brd 192.168.20.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet6 fe80::761e:cabc:c27a:4cd4/64 scope link noprefixroute valid_lft forever preferred_lft forever 3: virbr0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default qlen 1000 link/ether 52:54:00:c1:04:69 brd ff:ff:ff:ff:ff:ff inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0 valid_lft forever preferred_lft forever 4: virbr0-nic: <BROADCAST,MULTICAST> mtu 1500 qdisc pfifo_fast master virbr0 state DOWN group default qlen 1000 link/ether 52:54:00:c1:04:69 brd ff:ff:ff:ff:ff:ff//验证地址漂移（lb01中使用pkill nginx，再在lb02中使用ip a 查看） //恢复操作（在lb01中先启动nginx服务，再启动keepalived服务） //nginx站点/usr/share/nginx/html
//在node01和node02上操作 //开始修改node节点配置文件统一VIP（bootstrap.kubeconfig,kubelet.kubeconfig）
vim /opt/kubernetes/cfg/bootstrap.kubeconfig vim /opt/kubernetes/cfg/kubelet.kubeconfig vim /opt/kubernetes/cfg/kube-proxy.kubeconfig//统统修改为VIP
server: https://192.168.20.111:6443 systemctl restart kubelet.service systemctl restart kube-proxy.service//替换完成直接自检
cd /opt/kubernetes/cfg/ grep 111 * [root@node1 ~]# cd /opt/kubernetes/cfg/ [root@node1 cfg]# grep 111 * bootstrap.kubeconfig: server: https://192.168.20.111:6443 kubelet.kubeconfig: server: https://192.168.20.111:6443 kube-proxy.kubeconfig: server: https://192.168.20.111:6443 [root@node2 ~]# cd /opt/kubernetes/cfg/ [root@node2 cfg]# grep 111 * bootstrap.kubeconfig: server: https://192.168.20.111:6443 kubelet.kubeconfig: server: https://192.168.20.111:6443 kube-proxy.kubeconfig: server: https://192.168.20.111:6443//在lb01上 //查看nginx的k8s日志
[root@lb01 ~]# tail /var/log/nginx/k8s-access.log 192.168.20.20 192.168.20.40:6443 - [08/Oct/2020:00:23:11 +0800] 200 1119 192.168.20.20 192.168.20.40:6443 - [08/Oct/2020:00:23:11 +0800] 200 1118 192.168.20.30 192.168.20.40:6443 - [08/Oct/2020:00:23:16 +0800] 200 1120 192.168.20.30 192.168.20.40:6443 - [08/Oct/2020:00:23:16 +0800] 200 1120//在master01上操作 //测试创建pod
kubectl run nginx --image=nginx kubectl run --generator=deployment/apps.v1beta1 is DEPRECATED and will be removed in a future version. Use kubectl create instead. deployment.apps/nginx created//查看状态
[root@master ~]# kubectl get pods NAME READY STATUS RESTARTS AGE nginx-dbddb74b8-s4rdt 0/1 ContainerCreating 0 33s //正在创建中 [root@master2 cfg]# kubectl get pods NAME READY STATUS RESTARTS AGE nginx-dbddb74b8-s4rdt 1/1 Running 0 80s //创建完成，运行中//注意日志问题
[root@master ~]# kubectl logs nginx-dbddb74b8-s4rdt Error from server (Forbidden): Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy) ( pods/log nginx-dbddb74b8-s4rdt) [root@master ~]# kubectl create clusterrolebinding cluster-system-anonymous --clusterrole=cluster-admin --user=system:anonymous clusterrolebinding.rbac.authorization.k8s.io/cluster-system-anonymous created//查看pod网络
[root@master ~]# kubectl get pods -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE nginx-dbddb74b8-s4rdt 1/1 Running 0 4m18s 172.17.39.3 192.168.20.20 <none>//在对应网段的node节点上操作可以直接访问
[root@node1 cfg]# curl 172.17.39.3 <!DOCTYPE html> <html> <head> <title>Welcome to nginx!</title> <style> body { width: 35em; margin: 0 auto; font-family: Tahoma, Verdana, Arial, sans-serif; } </style> </head> <body> <h1>Welcome to nginx!</h1> <p>If you see this page, the nginx web server is successfully installed and working. Further configuration is required.</p> <p>For online documentation and support please refer to <a href="http://nginx.org/">nginx.org</a>.<br/> Commercial support is available at <a href="http://nginx.com/">nginx.com</a>.</p> <p><em>Thank you for using nginx.</em></p> </body> </html>//访问就会产生日志 //回到master01操作
[root@master ~]# kubectl logs nginx-dbddb74b8-s4rdt /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh 10-listen-on-ipv6-by-default.sh: Getting the checksum of /etc/nginx/conf.d/default.conf 10-listen-on-ipv6-by-default.sh: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh /docker-entrypoint.sh: Configuration complete; ready for start up 172.17.39.1 - - [07/Oct/2020:08:31:26 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.29.0" "-"