>
Coding
>
Kubernetes Recipes学习笔记
安装 Kubernetes 的步骤:
在所有节点上安装 Docker
添加仓库密钥
下载 Kubernetes 二进制文件
添加仓库源
安装工具:kubeadm kubelet kubectl
kubeadm init
添加工作节点的密钥
对用户应用权限
配置 CNI(容器网络接口),有
flannel
、Contiv 和 Weave
需开启的端口(如开启了防火墙):
Kubernetes 集群安装
准备两台主机/虚拟机:
# 国内可替换镜像源
sudo sed -i "s/archive.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list
sudo sed -i "s/security.ubuntu.com/mirrors.aliyun.com/g" /etc/apt/sources.list
apt-get update && apt-get upgrade -y
apt-get install docker.io -y
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
vi /etc/apt/sources.list.d/kubernetes.list
deb https://apt.kubernetes.io/ kubernetes-xenial main
apt-get update
apt install -y kubelet kubeadm kubectl
swapoff -a
# master
kubeadm init --pod-network-cidr=10.244.0.0/16 # 10.244.0.0/16为 flannel 配置的默认值
# 也可指定 IP:kubeadm init --apiserver-advertise-address=192.168.x.x --pod-network-cidr=10.244.0.0/16
# 出现错误可进行重置kubeadm reset
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# 国内可替换镜像源
sudo
sed
-
i
"s/archive.ubuntu.com/mirrors.aliyun.com/g"
/
etc
/
apt
/
sources
.
list
sudo
sed
-
i
"s/security.ubuntu.com/mirrors.aliyun.com/g"
/
etc
/
apt
/
sources
.
list
# 安装
apt
-
get
update
&&
apt
-
get
upgrade
-
y
apt
-
get
install
docker
.
io
-
y
curl
-
s
https
:
//packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
vi
/
etc
/
apt
/
sources
.
list
.
d
/
kubernetes
.
list
deb
https
:
//apt.kubernetes.io/ kubernetes-xenial main
apt
-
get
update
apt
install
-
y
kubelet
kubeadm
kubectl
swapoff
-
a
# master
kubeadm
init
--
pod
-
network
-
cidr
=
10.244.0.0
/
16
# 10.244.0.0/16为 flannel 配置的默认值
# 也可指定 IP:kubeadm init --apiserver-advertise-address=192.168.x.x --pod-network-cidr=10.244.0.0/16
# 出现错误可进行重置kubeadm reset
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 查看启动状况
kubectl get pods --all-namespaces
# 启动 flannel
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
mkdir
-
p
$
HOME
/
.
kube
sudo
cp
-
i
/
etc
/
kubernetes
/
admin
.
conf
$
HOME
/
.
kube
/
config
sudo
chown
$
(
id
-
u
)
:
$
(
id
-
g
)
$
HOME
/
.
kube
/
config
# 查看启动状况
kubectl
get
pods
--
all
-
namespaces
# 启动 flannel
kubectl
apply
-
f
https
:
//raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
cat
>
/
etc
/
docker
/
daemon
.
json
<
<
EOF
{
"exec-opts"
:
[
"native.cgroupdriver=systemd"
]
,
"log-driver"
:
"json-file"
,
"log-opts"
:
{
"max-size"
:
"100m"
}
,
"storage-driver"
:
"overlay2"
}
EOF
mkdir
-
p
/
etc
/
systemd
/
system
/
docker
.
service
.
d
# Restart docker.
systemctl
daemon
-
reload
systemctl
restart
docker
apiVersion: apps/v1 <em># for versions before 1.9.0 use apps/v1beta2</em>
kind: Deployment
metadata:
name: nginx-dep
spec:
selector:
matchLabels:
app: nginx
replicas: 2
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
apiVersion
:
apps
/
v1
<
em
>
# for versions before 1.9.0 use apps/v1beta2</em>
kind
:
Deployment
metadata
:
name
:
nginx
-
dep
spec
:
selector
:
matchLabels
:
app
:
nginx
replicas
:
2
template
:
metadata
:
labels
:
app
:
nginx
spec
:
containers
:
-
name
:
nginx
image
:
nginx
:
1.7.9
ports
:
-
containerPort
:
80
kubectl get deployments # 查看
kubectl describe deployment nginx-dep
kubectl get deployment nginx-dep -o yaml # 导出 yaml
kubectl set image deployment/nginx-dep nginx=nginx:1.8 # 升级 Nginx
kubectl rollout status deployment/nginx-dep # 查看状态
kubectl describe deployment nginx-dep # 再次查看确定Nginx版本
# 第二种升级方式:编辑nginx-dep.yml 中的 image 版本(如1.9.1)然后执行
kubectl apply -f nginx-dep.yaml
kubectl get deployments # 查看
kubectl describe deployment nginx-dep # 确认版本,其中也包含 revision的版本号(此处为3),如出现错误,可进行回滚
kubectl rollout history deployment/nginx-dep --revision=3 # 查看
kubectl rollout undo deployment/nginx-dep --to-revision=2 # 回滚
kubectl rollout status deployment/nginx-dep # 查看回滚状态
kubectl describe deployment nginx-dep # 再次确认
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
kubectl
get
deployments
# 查看
kubectl
describe
deployment
nginx
-
dep
kubectl
get
deployment
nginx
-
dep
-
o
yaml
# 导出 yaml
kubectl
set
image
deployment
/
nginx
-
dep
nginx
=
nginx
:
1.8
# 升级 Nginx
kubectl
rollout
status
deployment
/
nginx
-
dep
# 查看状态
kubectl
describe
deployment
nginx
-
dep
# 再次查看确定Nginx版本
# 第二种升级方式:编辑nginx-dep.yml 中的 image 版本(如1.9.1)然后执行
kubectl
apply
-
f
nginx
-
dep
.
yaml
kubectl
get
deployments
# 查看
kubectl
describe
deployment
nginx
-
dep
# 确认版本,其中也包含 revision的版本号(此处为3),如出现错误,可进行回滚
# 回滚
kubectl
rollout
history
deployment
/
nginx
-
dep
--
revision
=
3
# 查看
kubectl
rollout
undo
deployment
/
nginx
-
dep
--
to
-
revision
=
2
# 回滚
kubectl
rollout
status
deployment
/
nginx
-
dep
# 查看回滚状态
kubectl
describe
deployment
nginx
-
dep
# 再次确认
kubectl expose deployment nginx-dep --type="NodePort" --port 80
kubectl get services # 查看服务
kubectl describe services nginx-dep # 查看具体服务
# 此时使用 worker 节点的 IP,加NodePort 即可在浏览器中进行访问,如 http://192.168.0.12:30523
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
kubectl get pods --all-namespaces # 查看状态
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
apiVersion
:
v1
kind
:
Service
metadata
:
name
:
ingress
-
nginx
namespace
:
ingress
-
nginx
labels
:
app
.
kubernetes
.
io
/
name
:
ingress
-
nginx
app
.
kubernetes
.
io
/
part
-
of
:
ingress
-
nginx
spec
:
type
:
NodePort
ports
:
-
name
:
http
-
new
port
:
80
targetPort
:
80
protocol
:
TCP
nodePort
:
30090
selector
:
app
.
kubernetes
.
io
/
name
:
ingress
-
nginx
app
.
kubernetes
.
io
/
part
-
of
:
ingress
-
nginx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
apiVersion
:
extensions
/
v1beta1
kind
:
Ingress
metadata
:
name
:
ingress
-
test
annotations
:
kubernetes
.
io
/
ingress
.
class
:
nginx
ingress
.
kubernetes
.
io
/
rewrite
-
target
:
/
spec
:
rules
:
-
host
:
test
.
kuber
.
net
http
:
paths
:
-
path
:
/
backend
:
serviceName
:
web
-
01
servicePort
:
80
1
2
3
4
5
6
7
8
9
10
11
12
13
14
kind
:
Service
apiVersion
:
v1
metadata
:
name
:
loadbalancer
-
service
spec
:
selector
:
app
:
app
-
lb
ports
:
-
protocol
:
TCP
port
:
80
targetPort
:
9376
clusterIP
:
<
internalIP
>
loadBalancerIP
:
<
externalIP
>
type
:
LoadBalancer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
vagrant
@
master
-
node
:
~
$
clear
apiVersion
:
v1
kind
:
Pod
metadata
:
name
:
busybox
namespace
:
default
spec
:
containers
:
-
image
:
busybox
:
1.28
command
:
-
sleep
-
"3600"
imagePullPolicy
:
IfNotPresent
name
:
busybox
restartPolicy
:
Always
kubectl exec -it busybox -- nslookup kubernetes
# 出现问题排查思路
kubectl exec -it busybox -- cat /etc/resolv.conf
kubectl get pods -n kube-system
kubectl get services -n kube-system
kubectl get endpoints kube-dns -n kube-system
kubectl get nodes <worker-node> -o yaml # 修改名称查看statuses中的addresses
# 如为无法连接到的IP,则需进行修改
# 进入对应的工作节点
# vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# 在EnvironmentFile=-/etc/default/kubelet行之后添加
Environment="KUBELET_EXTRA_ARGS=--node-ip=<your.ip.addr>"
systemctl daemon-reload
systemctl restart kubelet
# 若依然有问题则打执行 kubeadm reset 重新加入主节点
kubectl
get
nodes
<
worker
-
node
>
-
o
yaml
# 修改名称查看statuses中的addresses
# 如为无法连接到的IP,则需进行修改
# 进入对应的工作节点
# vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
# 在EnvironmentFile=-/etc/default/kubelet行之后添加
Environment
=
"KUBELET_EXTRA_ARGS=--node-ip=<your.ip.addr>"
# 重启
systemctl
daemon
-
reload
systemctl
restart
kubelet
# 若依然有问题则打执行 kubeadm reset 重新加入主节点
chown nobody:nogroup general/
id nobody # 查看用户 id,稍后使用
# vi /etc/exports 可在其中添加多个 worker node
/var/nfs/general 192.168.x.x(rw,sync,no_subtree_check) 192.168.x.x(rw,sync,no_subtree_check)
exportfs -r # 不执行这条可能后续可能会失败并报mount.nfs: access denied by server while mounting...
apt
-
get
install
nfs
-
kernel
-
server
-
y
mkdir
/
var
/
nfs
/
general
-
p
cd
/
var
/
nfs
/
chown
nobody
:
nogroup
general
/
id
nobody
# 查看用户 id,稍后使用
# vi /etc/exports 可在其中添加多个 worker node
/
var
/
nfs
/
general
192.168.x.x
(
rw
,
sync
,
no_subtree_check
)
192.168.x.x
(
rw
,
sync
,
no_subtree_check
)
exportfs
-
r
# 不执行这条可能后续可能会失败并报mount.nfs: access denied by server while mounting...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
apiVersion
:
v1
kind
:
PersistentVolume
metadata
:
name
:
kube
-
pv
spec
:
capacity
:
storage
:
1Gi
volumeMode
:
Filesystem
accessModes
:
-
ReadWriteMany
persistentVolumeReclaimPolicy
:
Recycle
nfs
:
path
:
/
var
/
nfs
/
general
server
:
192.168.0.11
readOnly
:
false
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
apiVersion
:
v1
kind
:
Pod
metadata
:
name
:
nfs
-
pod
labels
:
name
:
nfs
-
pod
spec
:
containers
:
-
name
:
nfs
-
container
image
:
busybox
:
1.28
command
:
-
sleep
-
"3600"
volumeMounts
:
-
name
:
nfsvol
mountPath
:
/
tmp
restartPolicy
:
Always
securityContext
:
fsGroup
:
65534
runAsUser
:
65534
volumes
:
-
name
:
nfsvol
persistentVolumeClaim
:
claimName
:
kube
-
pvc
kubectl get pods -o wide # 此前建立 nginx-dep 有两个 pod 来运行中
kubectl delete pod nginx-dep-9f46bb5-bck7x # 删除其中的一个
kubectl get pods -o wide # 再次查看会发现 Kubernetes 正在重新创建一个
# 进入一个 worker 节点停止服务
service kubelet stop
# 几分钟后节点会显示为NotReady(kubectl get nodes)
kubectl get pods -o wide # 此时如有多个节点会自动在其它节点上创建Pod
kubectl
get
pods
-
o
wide
# 此前建立 nginx-dep 有两个 pod 来运行中
kubectl
delete
pod
nginx
-
dep
-
9f46bb5
-
bck7x
# 删除其中的一个
kubectl
get
pods
-
o
wide
# 再次查看会发现 Kubernetes 正在重新创建一个
# 进入一个 worker 节点停止服务
service
kubelet
stop
# 几分钟后节点会显示为NotReady(kubectl get nodes)
kubectl
get
pods
-
o
wide
# 此时如有多个节点会自动在其它节点上创建Pod
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
apiVersion
:
v1
kind
:
Pod
metadata
:
labels
:
test
:
liveness
name
:
liveness
-
exec
spec
:
containers
:
-
name
:
liveness
image
:
k8s
.
gcr
.
io
/
busybox
args
:
-
/
bin
/
sh
-
-
c
-
touch
/
tmp
/
healthy
;
sleep
30
;
rm
-
rf
/
tmp
/
healthy
;
sleep
600
livenessProbe
:
exec
:
command
:
-
cat
-
/
tmp
/
healthy
initialDelaySeconds
:
5
periodSeconds
:
5
git clone https://github.com/kubernetes-incubator/metrics-server
kubectl create -f metrics-server/deploy/1.8+/ # 同时对该目录下的所有 yaml 文件进行创建
kubectl get pods --all-namespaces # 查看启动状态
kubectl get --raw /apis/metrics.k8s.io/v1beta1
kubectl top nodes
# 横向扩展示例
kubectl run php-apache --image=k8s.gcr.io/hpa-example --requests=cpu=200m --expose --port=80
kubectl autoscale deployment php-apache --cpu-percent=50 --min=1 --max=4
kubectl get hpa -w # 查看
git
clone
https
:
//github.com/kubernetes-incubator/metrics-server
kubectl
create
-
f
metrics
-
server
/
deploy
/
1.8
+/
# 同时对该目录下的所有 yaml 文件进行创建
kubectl
get
pods
--
all
-
namespaces
# 查看启动状态
kubectl
get
--
raw
/
apis
/
metrics
.
k8s
.
io
/
v1beta1
kubectl
top
nodes
# 横向扩展示例
kubectl
run
php
-
apache
--
image
=
k8s
.
gcr
.
io
/
hpa
-
example
--
requests
=
cpu
=
200m
--
expose
--
port
=
80
kubectl
autoscale
deployment
php
-
apache
--
cpu
-
percent
=
50
--
min
=
1
--
max
=
4
kubectl
get
hpa
-
w
# 查看
- name: metrics-server
#image: k8s.gcr.io/metrics-server-amd64:v0.3.5
image: mirrorgooglecontainers/metrics-server-amd64:v0.3.5 # 国内被墙的问题
# 以下均作出了修改
imagePullPolicy: IfNotPresent
command:
- /metrics-server
- --kubelet-insecure-tls
- --kubelet-preferred-address-types=InternalIP
containers
:
-
name
:
metrics
-
server
#image: k8s.gcr.io/metrics-server-amd64:v0.3.5
image
:
mirrorgooglecontainers
/
metrics
-
server
-
amd64
:
v0
.
3.5
# 国内被墙的问题
# 以下均作出了修改
imagePullPolicy
:
IfNotPresent
command
:
-
/
metrics
-
server
-
--
kubelet
-
insecure
-
tls
-
--
kubelet
-
preferred
-
address
-
types
=
InternalIP
curl -L https://git.io/get_helm.sh | bash
helm init --wait
kubectl --namespace=kube-system create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default
helm ls # 验证,无报错信息即为正常
git clone https://github.com/kubernetes/charts
# vi prometheus-values.yml(仅测试使用)
alertmanager:
persistentVolume:
enabled: false
server:
persistentVolume:
enabled: false
helm install -f prometheus-values.yml charts/stable/prometheus --name prometheus --namespace prometheus
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
curl
-
L
https
:
//git.io/get_helm.sh | bash
helm
init
--
wait
kubectl
--
namespace
=
kube
-
system
create
clusterrolebinding
add
-
on
-
cluster
-
admin
--
clusterrole
=
cluster
-
admin
--
serviceaccount
=
kube
-
system
:
default
helm
ls
# 验证,无报错信息即为正常
git
clone
https
:
//github.com/kubernetes/charts
# vi prometheus-values.yml(仅测试使用)
alertmanager
:
persistentVolume
:
enabled
:
false
server
:
persistentVolume
:
enabled
:
false
helm
install
-
f
prometheus
-
values
.
yml
charts
/
stable
/
prometheus
--
name
prometheus
--
namespace
prometheus