kube-prometheus监控Linux主机

参考 https://blog.51cto.com/u_11880730/11223693

https://github.com/prometheus/node_exporter

tar zxvf node_exporter-1.8.2.linux-amd64.tar.gz -C /usr/local/
cd  /usr/local/
mv node_exporter-1.0.1.linux-amd64 node_exporter

vim /usr/lib/systemd/system/node_exporter.service

[Unit]
Description=node_exporter
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/node_exporter/node_exporter

[Install]
WantedBy=multi-user.target

http://172.17.13.1:9100/metrics

vim external-nodeexporter-service.yaml

apiVersion: v1
kind: Endpoints
metadata:
  labels:
    app.kubernetes.io/name: external-node-exporter    #endpoints 资源Labels标签
  name: external-node-exporter #endpoints 名称
  namespace: monitoring   #所属命名空间
subsets:  #用于指定一个或者多个IP，代表后端服务实例的地址
- addresses:
  - ip: 172.17.13.1
  - ip: 172.17.13.2
  - ip: 172.17.13.3
  - ip: 172.17.13.4
  - ip: 172.17.13.6
  - ip: 172.17.13.7
  - ip: 172.17.13.8
  - ip: 172.17.13.9
  - ip: 172.17.13.13
  ports:        #描述IP所使用的端口号信息
  - name: http-metrics
    port: 9100
    protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/name: external-node-exporter #同样是用于指定svc资源的标签Labels
  name: external-node-exporter #定义svc资源的名称
  namespace: monitoring   #定义所属命名空间
spec:
  ports: #定义了Svc所监控的端口以及关联的后端Pod端口信息
  - name: http-metrics   #定义了端口的名称
    port: 9100  #定义了svc所暴露出来的端口
    protocol: TCP  #定义了端口使用的协议
    targetPort: 9100  #定义了与svc所关联的Pod端口，该端口一般是Pod容器服务的内部端口
  sessionAffinity: None  #定义了会话亲和力类型，表示不启用亲和性，即新的请求可能会被路由到不同的后端Pod上，而不考虑之前请求的路由情况
  type: ClusterIP   #定义Svc访问类型，该类型只能在集群内部使用，适合作为后端服务的访问入口

vim external-nodeexporter-serviceMonitor.yaml

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor # prometheus-operator 定义的CRD
metadata:
  name: external-node-exporter 
  namespace: monitoring
  labels:
    k8s-apps: external-node-exporter
spec:
  jobLabel: app.kubernetes.io/name 
  selector:
    matchLabels:
      app.kubernetes.io/name: external-node-exporter
  namespaceSelector:
    matchNames:  # 配置需要自动发现的命名空间，可以配置多个
    - monitoring
  endpoints:
  - port: http-metrics # 拉去metric的端口，这个写的是 service的端口名称，即 service yaml的spec.ports.name
    interval: 15s # 拉取metric的时间间隔

vim external-nodeexporter-prometheusRule.yaml
需注意复用的规则应将监控目标修改成外部的node Job名称，上述job名称为external-node-exporter。那么该规则同样要指定该job名称，否则监控规则无法生效

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app.kubernetes.io/component: exporter
    app.kubernetes.io/name: external-node-exporter
    app.kubernetes.io/part-of: kube-prometheus
    app.kubernetes.io/version: 1.8.2
    prometheus: k8s
    role: alert-rules
  name: external-node-exporter-rules
  namespace: monitoring
spec:
  groups:
  - name: external-node-exporter
    rules:
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 24 hours.
      expr: |
        (
          node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 15
        and
          predict_linear(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 4 hours.
      expr: |
        (
          node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 10
        and
          predict_linear(node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 5% space left.
      expr: |
        (
          node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 5
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 30m
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 3% space left.
      expr: |
        (
          node_filesystem_avail_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 3
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 30m
      labels:
        severity: critical
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 24 hours.
      expr: |
        (
          node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 40
        and
          predict_linear(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 4 hours.
      expr: |
        (
          node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 20
        and
          predict_linear(node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 5% inodes left.
      expr: |
        (
          node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 5
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 3% inodes left.
      expr: |
        (
          node_filesystem_files_free{job="external-node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="external-node-exporter",fstype!="",mountpoint!=""} * 100 < 3
        and
          node_filesystem_readonly{job="external-node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeNetworkReceiveErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
        summary: Network interface is reporting many receive errors.
      expr: |
        rate(node_network_receive_errs_total{job="external-node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="external-node-exporter"}[2m]) > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeNetworkTransmitErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
        summary: Network interface is reporting many transmit errors.
      expr: |
        rate(node_network_transmit_errs_total{job="external-node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="external-node-exporter"}[2m]) > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeHighNumberConntrackEntriesUsed
      annotations:
        description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
        summary: Number of conntrack are getting close to the limit.
      expr: |
        (node_nf_conntrack_entries{job="external-node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
      labels:
        severity: warning
    - alert: NodeTextFileCollectorScrapeError
      annotations:
        description: Node Exporter text file collector on {{ $labels.instance }} failed to scrape.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
        summary: Node Exporter text file collector failed to scrape.
      expr: |
        node_textfile_scrape_error{job="external-node-exporter"} == 1
      labels:
        severity: warning
    - alert: NodeClockSkewDetected
      annotations:
        description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
        summary: Clock skew detected.
      expr: |
        (
          node_timex_offset_seconds{job="external-node-exporter"} > 0.05
        and
          deriv(node_timex_offset_seconds{job="external-node-exporter"}[5m]) >= 0
        )
        or
        (
          node_timex_offset_seconds{job="external-node-exporter"} < -0.05
        and
          deriv(node_timex_offset_seconds{job="external-node-exporter"}[5m]) <= 0
        )
      for: 10m
      labels:
        severity: warning
    - alert: NodeRAIDDegraded
      annotations:
        description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
        summary: RAID Array is degraded.
      expr: |
        node_md_disks_required{job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
      for: 15m
      labels:
        severity: critical
    - alert: NodeRAIDDiskFailure
      annotations:
        description: At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
        summary: Failed device in RAID array.
      expr: |
        node_md_disks{state="failed",job="external-node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: |
        (
          node_filefd_allocated{job="external-node-exporter"} * 100 / node_filefd_maximum{job="external-node-exporter"} > 70
        )
      for: 15m
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: |
        (
          node_filefd_allocated{job="external-node-exporter"} * 100 / node_filefd_maximum{job="external-node-exporter"} > 90
        )
      for: 15m
      labels:
        severity: critical
    - alert: NodeCPUHighUsage
      annotations:
        description: |
          CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
        summary: High CPU usage.
      expr: |
        sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="external-node-exporter", mode!="idle"}[2m]))) * 100 > 90
      for: 15m
      labels:
        severity: info
    - alert: NodeSystemSaturation
      annotations:
        description: |
          System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
          This might indicate this instance resources saturation and can cause it becoming unresponsive.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
        summary: System saturated, load per core is very high.
      expr: |
        node_load1{job="external-node-exporter"}
        / count without (cpu, mode) (node_cpu_seconds_total{job="external-node-exporter", mode="idle"}) > 2
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryMajorPagesFaults
      annotations:
        description: |
          Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.
          Please check that there is enough memory available at this instance.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
        summary: Memory major page faults are occurring at very high rate.
      expr: |
        rate(node_vmstat_pgmajfault{job="external-node-exporter"}[5m]) > 500
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryHighUtilization
      annotations:
        description: |
          Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
        summary: Host is running out of memory.
      expr: |
        100 - (node_memory_MemAvailable_bytes{job="external-node-exporter"} / node_memory_MemTotal_bytes{job="external-node-exporter"} * 100) > 90
      for: 15m
      labels:
        severity: warning
    - alert: NodeDiskIOSaturation
      annotations:
        description: |
          Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.
          This symptom might indicate disk saturation.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
        summary: Disk IO queue is high.
      expr: |
        rate(node_disk_io_time_weighted_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
      for: 30m
      labels:
        severity: warning
    - alert: NodeSystemdServiceFailed
      annotations:
        description: Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
        summary: Systemd service has entered failed state.
      expr: |
        node_systemd_unit_state{job="external-node-exporter", state="failed"} == 1
      for: 5m
      labels:
        severity: warning
    - alert: NodeBondingDegraded
      annotations:
        description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
        summary: Bonding interface is degraded
      expr: |
        (node_bonding_slaves - node_bonding_active) != 0
      for: 5m
      labels:
        severity: warning
  - name: external-external-node-exporter.rules
    rules:
    - expr: |
        count without (cpu, mode) (
          node_cpu_seconds_total{job="external-node-exporter",mode="idle"}
        )
      record: instance:node_num_cpu:sum
    - expr: |
        1 - avg without (cpu) (
          sum without (mode) (rate(node_cpu_seconds_total{job="external-node-exporter", mode=~"idle|iowait|steal"}[5m]))
        )
      record: instance:node_cpu_utilisation:rate5m
    - expr: |
        (
          node_load1{job="external-node-exporter"}
        /
          instance:node_num_cpu:sum{job="external-node-exporter"}
        )
      record: instance:node_load1_per_cpu:ratio
    - expr: |
        1 - (
          (
            node_memory_MemAvailable_bytes{job="external-node-exporter"}
            or
            (
              node_memory_Buffers_bytes{job="external-node-exporter"}
              +
              node_memory_Cached_bytes{job="external-node-exporter"}
              +
              node_memory_MemFree_bytes{job="external-node-exporter"}
              +
              node_memory_Slab_bytes{job="external-node-exporter"}
            )
          )
        /
          node_memory_MemTotal_bytes{job="external-node-exporter"}
        )
      record: instance:node_memory_utilisation:ratio
    - expr: |
        rate(node_vmstat_pgmajfault{job="external-node-exporter"}[5m])
      record: instance:node_vmstat_pgmajfault:rate5m
    - expr: |
        rate(node_disk_io_time_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      record: instance_device:node_disk_io_time_seconds:rate5m
    - expr: |
        rate(node_disk_io_time_weighted_seconds_total{job="external-node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      record: instance_device:node_disk_io_time_weighted_seconds:rate5m
    - expr: |
        sum without (device) (
          rate(node_network_receive_bytes_total{job="external-node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_receive_bytes_excluding_lo:rate5m
    - expr: |
        sum without (device) (
          rate(node_network_transmit_bytes_total{job="external-node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_transmit_bytes_excluding_lo:rate5m
    - expr: |
        sum without (device) (
          rate(node_network_receive_drop_total{job="external-node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_receive_drop_excluding_lo:rate5m
    - expr: |
        sum without (device) (
          rate(node_network_transmit_drop_total{job="external-node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_transmit_drop_excluding_lo:rate5m

grafana
Node 模版ID1860