每期一个小窍门 k8s版本的 Prometheus + grafana + alertmanager 三件套部署监控落地

首先部署prometheus

首先是pvc

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: prometheus-data-pvc
  namespace: monitor
spec:
  accessModes:
    - ReadWriteMany
  storageClassName: "data-nfs-storage"
  resources:
    requests:
      storage: 10Gi

然后接着 cluster-role 这里给了cluster-admin权限

apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus2
  namespace: monitor
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus2
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: prometheus2
  namespace: monitor 

由于用的是istio 所以这里的ingress用vs替代

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitor
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s
      external_labels:
        cluster: "kubernetes"

    alerting:
      alertmanagers:
      - static_configs:
        - targets: ["aalertmanager:9093"] 

    scrape_configs:
    - job_name: prometheus
      static_configs:
      - targets: ['localhost:9090']
        labels:
          instance: prometheus
          
    - job_name: kubelet
      metrics_path: /metrics/cadvisor
      scheme: https
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: replace
        target_label: endpoint
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace

    - job_name: k8s-state-metrics
      kubernetes_sd_configs: # k8s service discovery conf 服务发现配置
      - role: endpoints      # 去k8s的APIServer里拿取endpoints资源清单
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_label_kubernetes_io_name]
        regex: kube-state-metrics
        action: keep
      - source_labels: [__meta_kubernetes_pod_ip]
        regex: (.+)
        target_label: __address__
        replacement: ${
   
   1}:8080
      - source_labels: [__meta_kubernetes_endpoints_name]
        action: replace
        target_label: endpoint
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: service
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace

    - job_name: ingress
      metrics_path: /metrics/cadvisor
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__address__]
        regex: '(.+):10250'
        target_label: __address__
        replacement: ${
   
   1}:10254

    - job_name: jx-mysql-master-37
      static_configs:
      - targets: ['10.0.40.3:9104']
        labels:
          instance: jx-mysql-master-36 
    ############ 指定告警规则文件路径位置 ###################
    rule_files:
    - /etc/prometheus/rules/*.rules
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-rules
  namespace: monitor
data:
  node-exporter.rules: |
    groups:
    
    - name: NodeExporter
    
      rules:
    
        - alert: HostOutOfMemory
          expr: '(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host out of memory (instance {
   
   {
   
    $labels.instance }})
            description: "Node memory is filling up (< 10% left)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostMemoryUnderMemoryPressure
          expr: '(rate(node_vmstat_pgmajfault[1m]) > 1000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host memory under memory pressure (instance {
   
   {
   
    $labels.instance }})
            description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostMemoryIsUnderutilized
          expr: '(100 - (rate(node_memory_MemAvailable_bytes[30m]) / node_memory_MemTotal_bytes * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 1w
          labels:
            severity: info
          annotations:
            summary: Host Memory is underutilized (instance {
   
   {
   
    $labels.instance }})
            description: "Node memory is < 20% for 1 week. Consider reducing memory space. (instance {
   
   { $labels.instance }})\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualNetworkThroughputIn
          expr: '(sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: Host unusual network throughput in (instance {
   
   {
   
    $labels.instance }})
            description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualNetworkThroughputOut
          expr: '(sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: Host unusual network throughput out (instance {
   
   {
   
    $labels.instance }})
            description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualDiskReadRate
          expr: '(sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: Host unusual disk read rate (instance {
   
   {
   
    $labels.instance }})
            description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualDiskWriteRate
          expr: '(sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host unusual disk write rate (instance {
   
   {
   
    $labels.instance }})
            description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostOutOfDiskSpace
          expr: '((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host out of disk space (instance {
   
   {
   
    $labels.instance }})
            description: "Disk is almost full (< 10% left)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostDiskWillFillIn24Hours
          expr: '((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host disk will fill in 24 hours (instance {
   
   {
   
    $labels.instance }})
            description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostOutOfInodes
          expr: '(node_filesystem_files_free / node_filesystem_files * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host out of inodes (instance {
   
   {
   
    $labels.instance }})
            description: "Disk is almost running out of available inodes (< 10% left)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostFilesystemDeviceError
          expr: 'node_filesystem_device_error == 1'
          for: 0m
          labels:
            severity: critical
          annotations:
            summary: Host filesystem device error (instance {
   
   {
   
    $labels.instance }})
            description: "{
   
   { $labels.instance }}: Device error with the {
   
   { $labels.mountpoint }} filesystem\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostInodesWillFillIn24Hours
          expr: '(node_filesystem_files_free / node_filesystem_files * 100 < 10 and predict_linear(node_filesystem_files_free[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host inodes will fill in 24 hours (instance {
   
   {
   
    $labels.instance }})
            description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualDiskReadLatency
          expr: '(rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host unusual disk read latency (instance {
   
   {
   
    $labels.instance }})
            description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualDiskWriteLatency
          expr: '(rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host unusual disk write latency (instance {
   
   {
   
    $labels.instance }})
            description: "Disk latency is growing (write operations > 100ms)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostHighCpuLoad
          expr: '(sum by (instance) (avg by (mode, instance) (rate(node_cpu_seconds_total{mode!="idle"}[2m]))) > 0.8) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: Host high CPU load (instance {
   
   {
   
    $labels.instance }})
            description: "CPU load is > 80%\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostCpuIsUnderutilized
          expr: '(100 - (rate(node_cpu_seconds_total{mode="idle"}[30m]) * 100) < 20) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 1w
          labels:
            severity: info
          annotations:
            summary: Host CPU is underutilized (instance {
   
   {
   
    $labels.instance }})
            description: "CPU load is < 20% for 1 week. Consider reducing the number of CPUs.\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostCpuStealNoisyNeighbor
          expr: '(avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: Host CPU steal noisy neighbor (instance {
   
   {
   
    $labels.instance }})
            description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostCpuHighIowait
          expr: '(avg by (instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100 > 10) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: Host CPU high iowait (instance {
   
   {
   
    $labels.instance }})
            description: "CPU iowait > 10%. A high iowait means that you are disk or network bound.\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostUnusualDiskIo
          expr: '(rate(node_disk_io_time_seconds_total[1m]) > 0.5) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: Host unusual disk IO (instance {
   
   {
   
    $labels.instance }})
            description: "Time spent in IO is too high on {
   
   { $labels.instance }}. Check storage for issues.\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostContextSwitching
          expr: '((rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 10000) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: Host context switching (instance {
   
   {
   
    $labels.instance }})
            description: "Context switching is growing on the node (> 10000 / s)\n  VALUE = {
   
   { $value }}\n  LABELS = {
   
   { $labels }}"
    
        - alert: HostSwapIsFillingUp
          expr: '((1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}'
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: Host swap is filling up (instance {
   
   {
   
    $labels.instance }