Kubernetes K8s Prometheus添加监控二进制安装的etcd集群

172.17.20.104   etcd1.guoguo.com
172.17.20.105   etcd2.guoguo.com
172.17.20.106   etcd3.guoguo.com

• 1.
• 2.
• 3.

[root@etcd2 ssl]# cd /etc/kubernetes/ssl/
[root@etcd2 ssl]# ll
total 12
-rw-r--r-- 1 root root 1350 Jun  4 00:34 ca.pem
-rw-r--r-- 1 root root 1679 Jun  4 00:34 etcd-key.pem
-rw-r--r-- 1 root root 1428 Jun  4 00:34 etcd.pem
#我是用kubeasz安装的 证书是放在这个位置的
[root@etcd2 ssl]# scp /etc/kubernetes/ssl/* 172.17.0.1:/apps/k8s/prometheus/etcd/secret/
#传到创建监控的主机

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.

root@guoguo-M5-Pro:/apps/k8s/prometheus/etcd/secret# curl --cacert ca.pem --cert etcd.pem --key etcd-key.pem https://172.17.20.106:2379/metrics | tail -10
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  166k    0  166k    0     0   9.9M      0 --:--:-- --:--:-- --:--:-- 10.1M
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes 1.8446744073709552e+19
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 3
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.

root@guoguo-M5-Pro:/# kubectl create secret -n monitoring generic etcd-secret --from-file=/apps/k8s/prometheus/etcd/secret/ca.pem --from-file=/apps/k8s/prometheus/etcd/secret/etcd.pem --from-file=/apps/k8s/prometheus/etcd/secret/etcd-key.pem --dry-run -o yaml > secret-etcd.yaml
W0825 19:10:57.388357  532305 helpers.go:557] --dry-run is deprecated and can be replaced with --dry-run=client.
#转换成yaml 然后创建


root@guoguo-M5-Pro:/# kubectl apply -f secret-etcd.yaml
secret/etcd-secret created
#创建

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.

root@guoguo-M5-Pro:/# kubectl edit -n monitoring prometheus k8s
......
    app.kubernetes.io/component: prometheus
    app.kubernetes.io/name: prometheus
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 2.26.0
    prometheus: k8s
  name: k8s
  namespace: monitoring
  resourceVersion: "10516499"
  uid: 167328a1-df5a-4249-a107-fc7189f9f20c
spec:
  secrets:  #先搜索一下 看有没有 secrets  没有就在spec.下面新建一个  然后加上上面创建的etcd-secret
  - etcd-secret  #添加的
  alerting:
    alertmanagers:
    - apiVersion: v2
      name: alertmanager-main
      namespace: monitoring
      port: web
......
......

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.

root@guoguo-M5-Pro:/# kubectl exec -it -n monitoring prometheus-k8s-0 -c prometheus -- sh
/prometheus $ ls /etc/prometheus/secrets/etcd-secret/
ca.pem        etcd-key.pem  etcd.pem
#这个目录是挂载的目录

• 1.
• 2.
• 3.
• 4.

root@guoguo-M5-Pro:/apps/k8s/prometheus/etcd# cat etcd-svc-endpoints-servicemonitor.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: etcd-k8s   #名字
  namespace: monitoring
  labels:
    app: etcd   #标签 这个无头svc 和endpoints 和ServiceMointor 三个要一致
spec:
  ports:
  - name: api-etcd    #定义名字  endpoints 和 ServiceMonitor 要引用
    port: 2379        #端口号
  type: ClusterIP
  clusterIP: None  #无头service
---
apiVersion: v1
kind: Endpoints
metadata:
  name: etcd-k8s   #名字要和无头svc一致
  namespace: monitoring
  labels:
    app: etcd   #上面标签
subsets:
- addresses:
  - ip: 172.17.20.104  #ETCD的IP
  - ip: 172.17.20.105
  - ip: 172.17.20.106
  ports:
  - name: api-etcd   #上面无头svc的 spec.ports.name的名字
    port: 2379       #etcd端口号
    protocol: TCP    #协议
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: etcd-monitor
  namespace: monitoring
  labels:
    app: etcd
spec:
  selector:
    matchLabels:
      app: etcd
  endpoints:
  - port: api-etcd   #名字
    scheme: HTTPS    #协议
    interval: 30s    #获取监控数据的频率
    tlsConfig:   #证书的目录 这三个证书 目录都是 上面prometheus pod挂载的目录  并不是宿主机的目录！！！切记！！！
      caFile: /etc/prometheus/secrets/etcd-secret/ca.pem
      certFile: /etc/prometheus/secrets/etcd-secret/etcd.pem
      keyFile: /etc/prometheus/secrets/etcd-secret/etcd-key.pem
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
      - monitoring

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.
• 7.
• 8.
• 9.
• 10.
• 11.
• 12.
• 13.
• 14.
• 15.
• 16.
• 17.
• 18.
• 19.
• 20.
• 21.
• 22.
• 23.
• 24.
• 25.
• 26.
• 27.
• 28.
• 29.
• 30.
• 31.
• 32.
• 33.
• 34.
• 35.
• 36.
• 37.
• 38.
• 39.
• 40.
• 41.
• 42.
• 43.
• 44.
• 45.
• 46.
• 47.
• 48.
• 49.
• 50.
• 51.
• 52.
• 53.
• 54.
• 55.
• 56.

serviceMonitor/monitoring/etcd-monitor/0 (3/3 up)

• 1.

root@guoguo-M5-Pro:/apps/k8s# kubectl get svc -n monitoring grafana
NAME      TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
grafana   NodePort   10.99.178.106   <none>        3000:23204/TCP   25h

• 1.
• 2.
• 3.

默认用户名admin 密码admin  首次登录需要修改密码


在grafana官网模板中心搜索etcd，下载这个json格式的模板文件
https://grafana.com/dashboards/3070
https://grafana.com/api/dashboards/3070/revisions/3/download

• 1.
• 2.
• 3.
• 4.
• 5.
• 6.