天天看点
返回

K8s二进制安装 多master 高可用

K8s二进制安装 多master 高可用

5台机器 2c4g

ip 机器名 安装软件
172.24.19.23 master01 etcd,apiserver,controller-manager,scheduler,kublet
172.24.19.24 master02 etcd,apiserver,controller-manager,scheduler,kublet
172.24.19.25 nginx01 etcd,nginx
172.26.174.233 worker01 kubelet,kube-proxy
172.19.127.191 worker02 kubelet,kube-proxy

全部机器执行

cp /etc/hosts /etc/hosts.bak

cat >> /etc/hosts << EOF

172.24.19.23 master01

172.24.19.24 master02

172.24.19.25 nginx01

172.26.174.233 worker01

172.19.127.191 worker02

EOF

关闭防火墙 和 selinux

systemctl stop firewalld
systemctl disable firewalld
sed -i 's/enforcing/disabled/' /etc/selinux/config

关闭swap

swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab

时间同步

yum install -y ntpdate
ntpdate time.windows.com

将桥接的 IPv4 流量传递到 iptables 的链

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

Master部署

etcd 的工作路径为 /opt/etcd

kubernetes 的工作路径为 /opt/kubernetes

部署 etcd 集群

下载 cfssl 工具并使其可执行

# ssh master01 


cd /usr/local/bin
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O cfssl-certinfo
chmod +x cfssl*

生成 CA 证书

# ssh master01 


mkdir -p /opt/work/etcd; cd /opt/work/etcd
cat > ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "k8s",
      "OU": "system"
    }
  ],
  "ca": {
    "expiry": "87600h"
  }
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*.pem

如下图，出现ca-key.pem ca.pem即成功

K8s二进制安装 多master 高可用

生成 etcd 的 CA 证书

etcd集群ip规划为

172.24.19.23

172.24.19.24

172.24.19.25

# ssh master01


cd /opt/work/etcd

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
          "signing",
          "key encipherment",
          "server auth",
          "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF
cat > etcd-csr.json << EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "172.24.19.23", 
    "172.24.19.24",
    "172.24.19.25"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "CN",
    "ST": "GuangDong",
    "L": "GuangZhou",
    "O": "k8s",
    "OU": "system"
  }]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
ls etcd*.pem

下载 etcd 需要的文件

cd /opt/work/etcd

wget https://github.com/etcd-io/etcd/releases/download/v3.4.14/etcd-v3.4.14-linux-amd64.tar.gz
tar zxf etcd-v3.4.14-linux-amd64.tar.gz

mkdir -p /opt/etcd/{bin,cfg,data,ssl,shell}; cd /opt/etcd
cp /opt/work/etcd/etcd-v3.4.14-linux-amd64/etcd* bin/; chmod +x bin/etcd*
cp /opt/work/etcd/*.pem ssl/

cat > cfg/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd1"
ETCD_DATA_DIR="/opt/etcd/data"
ETCD_LISTEN_PEER_URLS="https://172.24.19.23:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.24.19.23:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.24.19.23:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://172.24.19.23:2379"
ETCD_INITIAL_CLUSTER="etcd1=https://172.24.19.23:2380,etcd2=https://172.24.19.24:2380,etcd3=https://172.24.19.25:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
cat > etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=-/opt/etcd/cfg/etcd.conf
WorkingDirectory=/opt/etcd/
ExecStart=/usr/local/bin/etcd \\
  --cert-file=/opt/etcd/ssl/etcd.pem \\
  --key-file=/opt/etcd/ssl/etcd-key.pem \\
  --trusted-ca-file=/opt/etcd/ssl/ca.pem \\
  --peer-cert-file=/opt/etcd/ssl/etcd.pem \\
  --peer-key-file=/opt/etcd/ssl/etcd-key.pem \\
  --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \\
  --peer-client-cert-auth \\
  --client-cert-auth
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
cat > shell/init.sh << "EOF"
for i in $(ls /opt/etcd/bin/*); do ln -s $i /usr/local/bin/; done
ln -s /opt/etcd/etcd.service /usr/lib/systemd/system/
EOF
chmod +x shell/init.sh

拷贝到另外的 master02 机器上

为了方便做个ssh免密钥吧

#ssh master01 命令三下回车
ssh-keygen -t rsa
ssh-copy-id [email protected]
ssh-copy-id [email protected]
--------
           
scp -r /opt/etcd [email protected]:/opt/
scp -r /opt/etcd [email protected]:/opt/

在另外两台etcd机器里更改 etcd.conf 中的 ip 和 ETCD_NAME 值

# ssh master01 master02


vim /opt/etcd/cfg/etcd.conf
############################
#### 需要修改的
#ETCD_NAME
#ETCD_LISTEN_PEER_URLS
#ETCD_LISTEN_CLIENT_URLS
#ETCD_INITIAL_ADVERTISE_PEER_URLS
#ETCD_ADVERTISE_CLIENT_URLS
############################

如下图

K8s二进制安装 多master 高可用

执行初始化脚本完成初始化

# ssh master01 master02

cd /opt/etcd/shell/
./init.sh

启动 etcd 并设置开机启动

# ssh master01 master02
systemctl daemon-reload
systemctl start etcd
systemctl status etcd
systemctl enable etcd

查看 etcd 集群状态

ETCDCTL_API=3 /usr/local/bin/etcdctl --write-out=table --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/etcd.pem --key=/opt/etcd/ssl/etcd-key.pem --endpoints=https://172.24.19.23:2379,https://172.24.19.24:2379,https://172.24.19.25:2379 endpoint health

如下图，etcd搭建成功

K8s二进制安装 多master 高可用

安装部署 kube-apiserver、kube-controller-manager、kube-scheduler

下载二进制文件

# ssh master01


mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs,shell}
mkdir /opt/work/kubernetes; cd /opt/work/kubernetes
cp /opt/work/etcd/ca*.pem .
cp /opt/work/etcd/ca-config.json .

wget https://dl.k8s.io/v1.21.0-alpha.2/kubernetes-server-linux-amd64.tar.gz

tar zxf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-apiserver kube-controller-manager kube-scheduler kubectl /opt/kubernetes/bin/

生成 kube-apiserver 需要的文件，下面的是所有出现的ip,master,worker,nginx一个不能少

# ssh master01


cd /opt/work/kubernetes

cat > kube-apiserver-csr.json << EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "172.24.19.23",
    "172.24.19.24",
    "172.24.19.25",
    "172.26.174.233",
    "172.19.127.191",
    "172.24.19.100",
    "10.255.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver

cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

cat > kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
  --anonymous-auth=false \\
  --bind-address=172.24.19.23 \\
  --secure-port=6443 \\
  --advertise-address=172.24.19.23 \\
  --insecure-port=0 \\
  --authorization-mode=Node,RBAC \\
  --runtime-config=api/all=true \\
  --enable-bootstrap-token-auth \\
  --service-cluster-ip-range=10.255.0.0/16 \\
  --token-auth-file=/opt/kubernetes/token.csv \\
  --service-node-port-range=30000-50000 \\
  --tls-cert-file=/opt/kubernetes/ssl/kube-apiserver.pem  \\
  --tls-private-key-file=/opt/kubernetes/ssl/kube-apiserver-key.pem \\
  --client-ca-file=/opt/kubernetes/ssl/ca.pem \\
  --kubelet-client-certificate=/opt/kubernetes/ssl/kube-apiserver.pem \\
  --kubelet-client-key=/opt/kubernetes/ssl/kube-apiserver-key.pem \\
  --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  --service-account-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \\
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
  --etcd-cafile=/opt/etcd/ssl/ca.pem \\
  --etcd-certfile=/opt/etcd/ssl/etcd.pem \\
  --etcd-keyfile=/opt/etcd/ssl/etcd-key.pem \\
  --etcd-servers=https://172.24.19.23:2379,https://172.24.19.24:2379,https://172.24.19.25:2379 \\
  --enable-swagger-ui=true \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=30 \\
  --audit-log-maxbackup=3 \\
  --audit-log-maxsize=100 \\
  --audit-log-path=/opt/kubernetes/logs/kube-apiserver-audit.log \\
  --event-ttl=1h \\
  --alsologtostderr=true \\
  --logtostderr=false \\
  --log-dir=/opt/kubernetes/logs \\
  --v=4"
EOF
cat > kube-apiserver.service << "EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

cd /opt/kubernetes
cp /opt/work/kubernetes/*.pem ssl/
cp /opt/work/kubernetes/kube-apiserver.conf cfg/
cp /opt/work/kubernetes/kube-apiserver.service /opt/work/kubernetes/token.csv .

生成 kubectl 需要的文件

# ssh master01


cd /opt/work/kubernetes

cat > kubectl-csr.json << EOF
{
  "CN": "kubectl",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "system:masters",             
      "OU": "system"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubectl-csr.json | cfssljson -bare kubectl
cp kubectl*.pem /opt/kubernetes/ssl/

生成 kube-controller-manager 需要的文件

# ssh master01


cd /opt/work/kubernetes

cat > kube-controller-manager-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "172.24.19.23",
      "172.24.19.24"
     
    ],
    "names": [
      {
        "C": "CN",
        "ST": "GuangDong",
        "L": "GuangZhou",
        "O": "system:kube-controller-manager",
        "OU": "system"
      }
    ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

cat > kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--bind-address=127.0.0.1 \\
  --kubeconfig=/opt/kubernetes/kube-controller-manager.kubeconfig \\
  --service-cluster-ip-range=10.255.0.0/16 \\
  --cluster-name=kubernetes \\
  --cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
  --cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  --allocate-node-cidrs=true \\
  --cluster-cidr=10.0.0.0/16 \\
  --experimental-cluster-signing-duration=87600h \\
  --root-ca-file=/opt/kubernetes/ssl/ca.pem \\
  --service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  --leader-elect=true \\
  --feature-gates=RotateKubeletServerCertificate=true \\
  --controllers=*,bootstrapsigner,tokencleaner \\
  --horizontal-pod-autoscaler-use-rest-clients=true \\
  --horizontal-pod-autoscaler-sync-period=10s \\
  --tls-cert-file=/opt/kubernetes/ssl/kube-controller-manager.pem \\
  --tls-private-key-file=/opt/kubernetes/ssl/kube-controller-manager-key.pem \\
  --use-service-account-credentials=true \\
  --alsologtostderr=true \\
  --logtostderr=false \\
  --log-dir=/opt/kubernetes/logs \\
  --v=2"
EOF
cat > kube-controller-manager.service << "EOF"
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/usr/local/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

cp kube-controller*.pem /opt/kubernetes/ssl/
cp kube-controller-manager.conf /opt/kubernetes/cfg/
cp kube-controller-manager.service /opt/kubernetes/

生成 kube-scheduler 需要的文件

# ssh master01


cd /opt/work/kubernetes

cat > kube-scheduler-csr.json << EOF
{
    "CN": "system:kube-scheduler",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "172.24.19.23",
      "172.24.19.24"
      
    ],
    "names": [
      {
        "C": "CN",
        "ST": "GuangDong",
        "L": "GuangZhou",
        "O": "system:kube-scheduler",
        "OU": "system"
      }
    ]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

cat > kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--address=127.0.0.1 \\
--kubeconfig=/opt/kubernetes/kube-scheduler.kubeconfig \\
--leader-elect=true \\
--alsologtostderr=true \\
--logtostderr=false \\
--log-dir=/opt/kubernetes/logs \\
--v=2"
EOF
cat > kube-scheduler.service << "EOF"
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/usr/local/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

cp kube-scheduler*.pem /opt/kubernetes/ssl/
cp kube-scheduler.conf /opt/kubernetes/cfg/
cp kube-scheduler.service /opt/kubernetes/

增加初始化脚本

# ssh master01


cd /opt/kubernetes

cat > shell/init.sh << "EOF"
for i in $(ls /opt/kubernetes/bin/*); do ln -s $i /usr/local/bin/; done
for i in $(ls /opt/kubernetes/*.service); do ln -s $i /usr/lib/systemd/system/; done
EOF
cat > shell/init-config.sh << "EOF"
kubectl config set-cluster kubernetes --certificate-authority=/opt/etcd/ssl/ca.pem --embed-certs=true --server=https://172.24.19.25:6443 --kubeconfig=kubectl.config
kubectl config set-credentials kubectl --client-certificate=/opt/kubernetes/ssl/kubectl.pem --client-key=/opt/kubernetes/ssl/kubectl-key.pem --embed-certs=true --kubeconfig=kubectl.config
kubectl config set-context kubernetes --cluster=kubernetes --user=kubectl --kubeconfig=kubectl.config
kubectl config use-context kubernetes --kubeconfig=kubectl.config
kubectl config set-cluster kubernetes --certificate-authority=/opt/etcd/ssl/ca.pem --embed-certs=true --server=https://172.24.19.25:6443 --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-credentials system:kube-controller-manager --client-certificate=/opt/kubernetes/ssl/kube-controller-manager.pem --client-key=/opt/kubernetes/ssl/kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
kubectl config set-cluster kubernetes --certificate-authority=/opt/etcd/ssl/ca.pem --embed-certs=true --server=https://172.24.19.25:6443 --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-credentials system:kube-scheduler --client-certificate=/opt/kubernetes/ssl/kube-scheduler.pem --client-key=/opt/kubernetes/ssl/kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
EOF
chmod +x shell/*.sh

下载nginx

#ssh nginx01
cd /usr/local
yum -y install make zlib zlib-devel gcc-c++ libtool  openssl openssl-devel

wget http://nginx.org/download/nginx-1.19.1.tar.gz
解压重命名为nginx
cd /usr/local/nginx
#安装stream模块

./configure --prefix=/usr/local/nginx --with-stream
make & make install

vim conf/nginx.conf
#与http同级
stream {
    upstream api-server  {
    server 172.24.19.23:6443;
    server 172.24.19.24:6443;
}

server {
    listen  6443;
        proxy_pass   api-server;
        }
   }

启动nginx ,缺日志文件就去建一个

sbin/nginx -c conf/nginx.conf

拷贝到另外俩台 master 机器上

# ssh master01


scp -r /opt/kubernetes [email protected]:/opt/

执行初始化脚本完成初始化,其他master应修改kube-apiserver.conf的ip

# ssh master01 master02


cd /opt/kubernetes

vi cfg/kube-apiserver.conf

./shell/init.sh
./shell/init-config.sh

启动并设置开机启动

# ssh master01 master02

systemctl start kube-apiserver
systemctl start kube-controller-manager
systemctl start kube-scheduler

systemctl enable kube-apiserver
systemctl enable kube-controller-manager
systemctl enable kube-scheduler

部署 kubectl

# ssh master01 master02


cd /opt/kubernetes
mkdir ~/.kube
cp kubectl.config ~/.kube/config

生成 clusterrolebinding 使 kubectl 可用

# ssh master01 master02


kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

           
kubectl cluster-info
K8s二进制安装 多master 高可用

安装 Dokcer

worker 机器上一起运行

# ssh worker01 worker02

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo
yum install docker-ce -y

设置镜像加速

# ssh worker01 worker02


mkdir -p /etc/docker
cat > /etc/docker/daemon.json << EOF
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": [
        "https://registry.docker-cn.com",
        "https://docker.mirrors.ustc.edu.cn"
    ]
}
EOF

启动并设置开机启动

# ssh worker01 worker02


systemctl start docker
systemctl enable docker

部署 worker kubernetes

回到 master01 上生成 worker 机器需要的文件

# ssh master01


mkdir -p /opt/kubernetes-node/{bin,cfg,ssl,shell,kubelet/logs,kube-proxy/logs}
cd /opt/work/kubernetes/kubernetes/server/bin
cp kubelet kube-proxy /opt/kubernetes-node/bin/
cp /opt/kubernetes/ssl/ca*.pem /opt/kubernetes-node/ssl/

生成 kubelet 需要的文件

# ssh master01


cd /opt/kubernetes-node

kubectl config set-cluster kubernetes --certificate-authority=/opt/kubernetes-node/ssl/ca.pem --embed-certs=true --server=https://172.24.19.25:6443 --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-credentials kubelet-bootstrap --token=$(awk -F "," '{print $1}' /opt/kubernetes/token.csv) --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

cat > cfg/kubelet-config.yaml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 172.26.174.233
port: 10250
readOnlyPort: 10255
cgroupDriver: systemd
clusterDNS:
 - 10.255.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
  x509:
    clientCAFile: /opt/kubernetes-node/ssl/ca.pem
  webhook:
    enabled: true
    cacheTTL: 2m0s
  anonymous:
    enabled: false
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
EOF

cat > kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/opt/kubernetes-node/kubelet
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/opt/kubernetes-node/kubelet-bootstrap.kubeconfig \
  --cert-dir=/opt/kubernetes-node/ssl \
  --kubeconfig=/opt/kubernetes-node/kubelet.kubeconfig \
  --config=/opt/kubernetes-node/cfg/kubelet-config.yaml \
  --network-plugin=cni \
  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.2 \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/opt/kubernetes-node/kubelet/logs \
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

生成 kube-proxy 需要的文件

cd /opt/kubernetes-node
cp /opt/work/kubernetes/ca-config.json .

cat > kube-proxy-csr.json << EOF
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF
cfssl gencert -ca=ssl/ca.pem -ca-key=ssl/ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
mv *.pem ssl/

kubectl config set-cluster kubernetes --certificate-authority=ssl/ca.pem --embed-certs=true --server=https://172.24.19.25:6443 --kubeconfig=kube-proxy.kubeconfig
kubectl config set-credentials kube-proxy --client-certificate=ssl/kube-proxy.pem --client-key=ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

cat > cfg/kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 172.26.174.233
clientConnection:
  kubeconfig: /opt/kubernetes-node/kube-proxy.kubeconfig
clusterCIDR: 192.168.0.0/16
healthzBindAddress: 172.26.174.233:10256
kind: KubeProxyConfiguration
metricsBindAddress: 172.26.174.233:10249
mode: "ipvs"
EOF

cat > kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
WorkingDirectory=/opt/kubernetes-node/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --config=/opt/kubernetes-node/cfg/kube-proxy.yaml \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/opt/kubernetes-node/kube-proxy/logs \
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

增加初始化脚本

# ssh master01

cd /opt/kubernetes-node
mv kube-pro*csr* /opt/work/kubernetes/
rm -f ca-config.json

cat > shell/init.sh << "EOF"
for i in $(ls /opt/kubernetes-node/bin/*);do ln -s $i /usr/local/bin/;done
for i in $(ls /opt/kubernetes-node/*.service);do ln -s $i /usr/lib/systemd/system/;done
EOF
chmod +x shell/init.sh

拷贝到 worker 机器上

# ssh master01

scp -r /opt/kubernetes-node [email protected]:/opt/kubernetes-node
scp -r /opt/kubernetes-node [email protected]:/opt/kubernetes-node

执行初始化脚本完成初始化

# ssh  worker01 worker02


cd /opt/kubernetes-node
./shell/init.sh

启动并设置开机启动

# ssh worker01 worker02

cd /opt/kubernetes-node
#####################################################
########### xxxxx 是本机IP  
#####################################################
vi cfg/kube-proxy.yaml
vi cfg/kubelet-config.yaml

systemctl daemon-reload
systemctl start kubelet
systemctl start kube-proxy

systemctl status kubelet
systemctl status kube-proxy

systemctl enable kubelet
systemctl enable kube-proxy

查看并操作 worker 节点上线

#ssh master01
kubectl get csr
K8s二进制安装 多master 高可用
#上线
kubectl certificate approve ${NAME}

kubectl get csr
kubectl get nodes

如下则是正常的

K8s二进制安装 多master 高可用

部署网络插件

#ALL


mkdir /opt/kubernetes/plugins; cd /opt/kubernetes/plugins

wget https://docs.projectcalico.org/v3.17/manifests/calico.yaml
kubectl apply -f calico.yaml

查看进度

kubectl get pods -A
K8s二进制安装 多master 高可用

部署 coredns

#ssh master01


cd /opt/kubernetes/plugins

1

cat > coredns.yaml << "EOF"
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  - namespaces
  verbs:
  - list
  - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "CoreDNS"
spec:
  # replicas: not specified here:
  # 1. Default is 1.
  # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      tolerations:
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      nodeSelector:
        kubernetes.io/os: linux
      affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
             podAffinityTerm:
               labelSelector:
                 matchExpressions:
                   - key: k8s-app
                     operator: In
                     values: ["kube-dns"]
               topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: coredns/coredns:1.8.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.255.0.2
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP
EOF
           
kubectl apply -f coredns.yaml

部署 nginx 应用试验结果

#ssh master01


mkdir /opt/kubernetes/workspace; cd /opt/kubernetes/workspace

cat > nginx.yaml << EOF
---
apiVersion: v1
kind: ReplicationController
metadata:
  name: nginx-controller
spec:
  replicas: 2
  selector:
    name: nginx
  template:
    metadata:
      labels:
        name: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.19.6
          ports:
            - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service-nodeport
spec:
  ports:
    - port: 80
      targetPort: 80
      nodePort: 30001
      protocol: TCP
  type: NodePort
  selector:
    name: nginx
EOF

kubectl apply -f nginx.yaml

kubectl get svc

kubectl get pods

k8s
上一篇: kubernetes（k8s）二进制部署——etcd部署一、 相关概述二、模拟实验
下一篇: ​LeetCode刷题实战357：计算各个位数不同的数字个数

继续阅读