K8s NetworkPolicy、LimitRange和ResourceQuota详解、K8s运行ZooKeeper,Mysql,Jenkins集群、K8s集群及应用监控Prometheus

原创

已于 2023-04-02 16:29:04 修改 · 4.2k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#kubernetes

于 2023-04-02 16:23:33 首次发布

文章展示了在Kubernetes平台上实施网络策略限制跨命名空间访问、部署Zookeeper集群并利用NFS和StorageClass实现持久化存储、定制Docker镜像及构建Java服务、LNMP环境下的WordPress部署，以及设置资源限制的全过程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1. 基于 NetworkPolicy 限制 magedu namespace 中的所有 pod 不能跨 namespace 访问 (只能访问当前 namespace 中的所有 pod)。

#在default下创建2个deploy, centos7-default和nginx1-default

root@k8s-master1:~/20230328# vim centos7-default.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: centos7-default
  name: centos7-default
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: centos7-default
  template:
    metadata:
      labels:
        app: centos7-default
    spec:
      containers:
      - image: centos:centos7.9.2009
        name: centos
        command:
          - sleep
          - "50000000"
root@k8s-master1:~/20230328# kubectl apply -f centos7-default.yaml 
deployment.apps/centos7-default created
root@k8s-master1:~/20230328# kubectl  create deploy nginx1-default --image=nginx
deployment.apps/nginx1-default created

#查看default下创建的deploy

root@k8s-master1:~/20230328# kubectl get pods --show-labels
NAME                               READY   STATUS    RESTARTS   AGE     LABELS
centos7-default-7cff9984c9-t7sdp   1/1     Running   0          9m50s   app=centos7-default,pod-template-hash=7cff9984c9
nginx1-default-76d65dfb67-gsdm5    1/1     Running   0          15s     app=nginx1-default,pod-template-hash=76d65dfb67
root@k8s-master1:~/20230328# kubectl get pods -o wide
NAME                               READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
centos7-default-7cff9984c9-t7sdp   1/1     Running   0          10m   10.200.218.71    192.168.7.113   <none>           <none>
nginx1-default-76d65dfb67-gsdm5    1/1     Running   0          75s   10.200.151.200   192.168.7.112   <none>           <none>

#创建namespace magedu，并在此空间下创建2个deploy

root@k8s-master1:~/20230328# cat centos7-magedu.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: centos7-magedu
  name: centos7-magedu
  namespace: magedu
spec:
  replicas: 1
  selector:
    matchLabels:
      app: centos7-magedu
  template:
    metadata:
      labels:
        app: centos7-magedu
    spec:
      containers:
      - image: centos:centos7.9.2009
        name: centos
        command:
          - sleep
          - "50000000"
root@k8s-master1:~/20230328# kubectl apply -f centos7-magedu.yaml 
deployment.apps/centos7-magedu created

root@k8s-master1:~/20230328# kubectl  create deploy nginx2-magedu --image=nginx --namespace magedu
deployment.apps/nginx2-magedu created

#查看magedu下创建的资源

root@k8s-master1:~/20230328# kubectl get pods -n magedu --show-labels
NAME                             READY   STATUS    RESTARTS   AGE     LABELS
centos7-magedu-bc6b4665f-9g6zh   1/1     Running   0          25m     app=centos7-magedu,pod-template-hash=bc6b4665f
nginx2-magedu-5ddc8898d6-v98v4   1/1     Running   0          7m54s   app=nginx2-magedu,pod-template-hash=5ddc8898d6
root@k8s-master1:~/20230328# kubectl get pods -n magedu -o wide
NAME                             READY   STATUS    RESTARTS   AGE     IP               NODE            NOMINATED NODE   READINESS GATES
centos7-magedu-bc6b4665f-9g6zh   1/1     Running   0          24m     10.200.218.1     192.168.7.111   <none>           <none>
nginx2-magedu-5ddc8898d6-v98v4   1/1     Running   0          7m45s   10.200.151.199   192.168.7.112   <none>           <none>

#进入default空间下centos7-default pod访问magedu空间下nginx2-magedu pod服务，可正常访问

root@k8s-master1:~/20230328# kubectl exec -it centos7-default-7cff9984c9-t7sdp bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@centos7-default-7cff9984c9-t7sdp /]# curl 10.200.151.199
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html {
   
    color-scheme: light dark; }
body {
   
    width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

#进入magedu空间下centos7-magedu的pod,访问default空间下nginx1-default服务，访问正常

root@k8s-master1:~/20230328# kubectl exec -it centos7-magedu-bc6b4665f-9g6zh bash -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@centos7-magedu-bc6b4665f-9g6zh /]# curl 10.200.151.200
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html {
   
    color-scheme: light dark; }
body {
   
    width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

#创建networkpolicy

root@k8s-master1:~# vi Egress-magedu.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: egress-access-networkpolicy
  namespace: magedu
spec:
  policyTypes:
  - Egress
  podSelector:
    matchLabels: {
   
   }
  egress:
    - to:
      - podSelector:
          matchLabels: {
   
   }
root@k8s-master1:~/20230328# kubectl apply -f Egress-magedu.yaml 
networkpolicy.networking.k8s.io/egress-access-networkpolicy created

#查看刚创建的networkpolicy

root@k8s-master1:~/20230328# kubectl get networkpolicy -n magedu
NAME                          POD-SELECTOR   AGE
egress-access-networkpolicy   <none>         17s

#列出default和magedu命名空间下的pod
root@k8s-master1:~/20230328# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
centos7-default-7cff9984c9-t7sdp 1/1 Running 0 3h46m 10.200.218.71 192.168.7.113
nginx1-default-76d65dfb67-gsdm5 1/1 Running 0 3h36m 10.200.151.200 192.168.7.112

root@k8s-master1:~/20230328# kubectl get pods -n magedu -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
centos7-magedu-bc6b4665f-9g6zh 1/1 Running 0 4h7m 10.200.218.1 192.168.7.111
nginx2-magedu-5ddc8898d6-v98v4 1/1 Running 0 3h50m 10.200.151.199 192.168.7.112

#default下pod能访问magedu空间下pod, 正常访问

root@k8s-master1:~/20230328# kubectl exec -it centos7-default-7cff9984c9-t7sdp bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@centos7-default-7cff9984c9-t7sdp /]# curl 10.200.151.199
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html {
   
    color-scheme: light dark; }
body {
   
    width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

#magedu下pod访问default空间下pod, 不能正常访问

root@k8s-master1:~/20230328# kubectl exec -it centos7-magedu-bc6b4665f-9g6zh bash -n magedu kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead. [root@centos7-magedu-bc6b4665f-9g6zh /]# curl 10.200.151.200 ^C

##magedu下pod访问同命名空间下pod, 正常访问

root@k8s-master1:~/20230328# kubectl exec -it centos7-magedu-bc6b4665f-9g6zh bash -n magedu
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@centos7-magedu-bc6b4665f-9g6zh /]# curl 10.200.151.199
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html {
   
    color-scheme: light dark; }
body {
   
    width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

总结：网络策略生效后，其它namespace可以访问magedu namespace内的服务，magedu内的pod无法访问外部的服务

2. 在 kubernetes 环境部署 zookeeper 集群并基于 NFS 或 StorageClass 等方式实现创建持久化。

#下载jdk8镜像

root@k8s-master1:/opt/k8s-data# docker pull elevy/slim_java:8
8: Pulling from elevy/slim_java
88286f41530e: Downloading 
7141511c4dad: Download complete 
fd529fe251b3: Download complete 
8: Pulling from elevy/slim_java
88286f41530e: Pull complete 
7141511c4dad: Pull complete 
fd529fe251b3: Pull complete 
Digest: sha256:044e42fb89cda51e83701349a9b79e8117300f4841511ed853f73caf7fc98a51
Status: Downloaded newer image for elevy/slim_java:8
docker.io/elevy/slim_java:8

#镜像重命名,打tag

root@k8s-master1:/opt/k8s-data# docker tag docker.io/elevy/slim_java:8 harbor.magedu.net/baseimages/slim_java:8
root@k8s-master1:/opt/k8s-data# docker login harbor.magedu.net
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

#push到本地镜像仓库

root@k8s-master1:/opt/k8s-data# docker push harbor.magedu.net/baseimages/slim_java:8
The push refers to repository [harbor.magedu.net/baseimages/slim_java]
e053edd72ca6: Pushed 
aba783efb1a4: Pushed 
5bef08742407: Pushed 
8: digest: sha256:817d0af5d4f16c29509b8397784f5d4ec3accb1bfde4e474244ed3be7f41a604 size: 952

##修改dockerfile 依赖镜像地址

root@k8s-master1:/opt/k8s-data# cd  dockerfile/web/magedu/zookeeper/
FROM harbor.magedu.net/baseimages/slim_java:8

##修改编译脚本镜像地址

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# vi build-command.sh
docker build -t harbor.magedu.net/magedu/zookeeper:${TAG} .
docker push  harbor.magedu.net/magedu/zookeeper:${TAG}

##编译并上传镜像

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# bash build-command.sh v3.4.14
5bef08742407: Mounted from baseimages/slim_java 
v3.4.14: digest: sha256:f10eb1634d0d2d5eae520c0b9b170c00ef9a209c3b614139bc3207073509987c size: 2621

#测试镜像

root@k8s-master1:/opt/k8s-data/dockerfile/web/magedu/zookeeper# docker run -it --rm harbor.magedu.net/magedu/zookeeper:v3.4.14
2023-03-28 14:28:46,666 [myid:] - INFO  [main:ServerCnxnFactory@117] - Using org.apache.zookeeper.server.NIOServerCnxnFactory as server connection factory
2023-03-28 14:28:46,679 [myid:] - INFO  [main:NIOServerCnxnFactory@89] - binding to port 0.0.0.0/0.0.0.0:2181

##NFS 服务器创建zookeeper的pv数据目录

root@haproxy1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-1
root@haproxy1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-2
root@haproxy1:~# mkdir -p /data/k8sdata/magedu/zookeeper-datadir-3

##NFS配置共享目录

root@haproxy1:~# vi /etc/exports
/data/k8sdata *(rw,no_root_squash,no_subtree_check)

##生效NFS配置

root@haproxy1:~# exportfs -r

##master1测试NFS服务

root@k8s-master1:~# showmount -e 172.31.7.109
Export list for 172.31.7.109:
/data/k8sdata *

#创建PV和PVC

root@k8s-master1:/opt/k8s-data/yaml/magedu/zookeeper/pv# kubectl apply -f .
persistentvolume/zookeeper-datadir-pv-1 created
persistentvolume/zookeeper-datadir-pv-2 created
persistentvolume/zookeeper-datadir-pv-3 created
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persisten