consul_exporter监控

# 1.下载 node_exporter

访问官网地址下载<https://prometheus.io/download/#node_exporter>

```bash
mkdir -p /opt/agent
cd /opt/agent
wget https://github.com/prometheus/node_exporter/releases/download/v1.1.2/node_exporter-1.1.2.linux-amd64.tar.gz
```

# 2.配置 systemd 管理

## 2.1 创建启动用户和用户组

```bash
useradd -M -s /sbin/nologin prometheus
```

## 2.2 创建node_exporter.service

```bash
# vim /etc/systemd/system/node-exporter.service

[Unit]
Description=node-export service agent by jonnyan404
Requires=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Restart=on-failure
ExecStart=/path/to/node_exporter --collector.tcpstat
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
TimeoutStopSec=5

[Install]
WantedBy=multi-user.target
```

## 2.3 设置开机自启并启动

```bash
systemctl enable node-exporter.service
systemctl start node-exporter.service
```

## 2.4 查看日志

```bash
journalctl -u node-exporter.service
```

# 3. 配置自动发现的主机列表

基于 file_sd_configs
有 yaml和json两种格式,这里我们采用yaml

- yaml格式

```yaml
# vim /opt/jonnyan404/prometheus/target/linux.yml 文件名字自己取
- targets: ['192.168.1.220:9100']
  labels:
    app:    'app1'
    env:   'game1'
    region: 'us-west-2'
- targets: ['192.168.1.221:9100']
  labels:
    app:    'app2'
    env:   'game2'
    region: 'ap-southeast-1'

```

- json格式

```json
[
  {
    "targets": [ "192.168.1.221:29090"],
    "labels": {
      "app": "app1",
      "env": "game1",
      "region": "us-west-2"
    }
  },
  {
    "targets": [ "192.168.1.222:29090" ],
    "labels": {
      "app": "app2",
      "env": "game2",
      "region": "ap-southeast-1"
    }
  }
]

```

# 4. 配置告警规则

- vim /opt/jonnyan404/prometheus/rules/node-exporter-record.yml

```yaml
groups:
  - name: node_exporter-record
    rules:
    - expr: up
      record: node_exporter:up
      labels:
        desc: "节点是否在线, 在线1,不在线0"
        unit: " "
        job: "aws_ec2"
    - expr: time() - node_boot_time_seconds{}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:node_uptime
      labels:
        desc: "节点的运行时间"
        unit: "s"
        job: "aws_ec2"
##############################################################################################
#                              cpu                                                           #
    - expr: (1 - avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode="idle"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:total:percent
      labels:
        desc: "节点的cpu总消耗百分比"
        unit: "%"
        job: "aws_ec2"

- expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode="idle"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:idle:percent
      labels:
        desc: "节点的cpu idle百分比"
        unit: "%"
        job: "aws_ec2"

- expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode="iowait"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:iowait:percent
      labels:
        desc: "节点的cpu iowait百分比"
        unit: "%"
        job: "aws_ec2"

- expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode="system"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:system:percent
      labels:
        desc: "节点的cpu system百分比"
        unit: "%"
        job: "aws_ec2"

- expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode="user"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:user:percent
      labels:
        desc: "节点的cpu user百分比"
        unit: "%"
        job: "aws_ec2"

- expr: (avg by (environment,instance) (irate(node_cpu_seconds_total{job="aws_ec2",mode=~"softirq|nice|irq|steal"}[5m])))  * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:other:percent
      labels:
        desc: "节点的cpu 其他的百分比"
        unit: "%"
        job: "aws_ec2"
##############################################################################################
#                                    memory                                                  #
    - expr: node_memory_MemTotal_bytes{job="aws_ec2"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:total
      labels:
        desc: "节点的内存总量"
        unit: byte
        job: "aws_ec2"

- expr: node_memory_MemFree_bytes{job="aws_ec2"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:free
      labels:
        desc: "节点的剩余内存量"
        unit: byte
        job: "aws_ec2"

- expr: node_memory_MemTotal_bytes{job="aws_ec2"} - node_memory_MemFree_bytes{job="aws_ec2"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:used
      labels:
        desc: "节点的已使用内存量"
        unit: byte
        job: "aws_ec2"

- expr: node_memory_MemTotal_bytes{job="aws_ec2"} - node_memory_MemAvailable_bytes{job="aws_ec2"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:actualused
      labels:
        desc: "节点用户实际使用的内存量"
        unit: byte
        job: "aws_ec2"

- expr: (1-(node_memory_MemAvailable_bytes{job="aws_ec2"} / (node_memory_MemTotal_bytes{job="aws_ec2"})))* 100* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:used:percent
      labels:
        desc: "节点的内存使用百分比"
        unit: "%"
        job: "aws_ec2"

- expr: ((node_memory_MemAvailable_bytes{job="aws_ec2"} / (node_memory_MemTotal_bytes{job="aws_ec2"})))* 100* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:memory:free:percent
      labels:
        desc: "节点的内存剩余百分比"
        unit: "%"
        job: "aws_ec2"
##############################################################################################
#                                   load                                                     #
    - expr: sum by (instance) (node_load1{job="aws_ec2"})* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:load:load1
      labels:
        desc: "系统1分钟负载"
        unit: " "
        job: "aws_ec2"

- expr: sum by (instance) (node_load5{job="aws_ec2"})* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:load:load5
      labels:
        desc: "系统5分钟负载"
        unit: " "
        job: "aws_ec2"

- expr: sum by (instance) (node_load15{job="aws_ec2"})* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:load:load15
      labels:
        desc: "系统15分钟负载"
        unit: " "
        job: "aws_ec2"

##############################################################################################
#                                 disk                                                       #
    - expr: node_filesystem_size_bytes{job="aws_ec2" ,fstype=~"ext4|xfs"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:usage:total
      labels:
        desc: "节点的磁盘总量"
        unit: byte
        job: "aws_ec2"

- expr: node_filesystem_avail_bytes{job="aws_ec2",fstype=~"ext4|xfs"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:usage:free
      labels:
        desc: "节点的磁盘剩余空间"
        unit: byte
        job: "aws_ec2"

- expr: node_filesystem_size_bytes{job="aws_ec2",fstype=~"ext4|xfs"} - node_filesystem_avail_bytes{job="aws_ec2",fstype=~"ext4|xfs"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:usage:used
      labels:
        desc: "节点的磁盘使用的空间"
        unit: byte
        job: "aws_ec2"

- expr:  (1 - node_filesystem_avail_bytes{job="aws_ec2",fstype=~"ext4|xfs"} / node_filesystem_size_bytes{job="aws_ec2",fstype=~"ext4|xfs"}) * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:used:percent
      labels:
        desc: "节点的磁盘的使用百分比"
        unit: "%"
        job: "aws_ec2"

- expr: irate(node_disk_reads_completed_total{job="aws_ec2"}[1m])* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:read:count:rate
      labels:
        desc: "节点的磁盘读取速率"
        unit: "次/秒"
        job: "aws_ec2"

- expr: irate(node_disk_writes_completed_total{job="aws_ec2"}[1m])* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:write:count:rate
      labels:
        desc: "节点的磁盘写入速率"
        unit: "次/秒"
        job: "aws_ec2"

- expr: (irate(node_disk_written_bytes_total{job="aws_ec2"}[1m]))/1024/1024* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:read:mb:rate
      labels:
        desc: "节点的设备读取MB速率"
        unit: "MB/s"
        job: "aws_ec2"

- expr: (irate(node_disk_read_bytes_total{job="aws_ec2"}[1m]))/1024/1024* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:disk:write:mb:rate
      labels:
        desc: "节点的设备写入MB速率"
        unit: "MB/s"
        job: "aws_ec2"

##############################################################################################
#                                filesystem                                                  #
    - expr:   (1 -node_filesystem_files_free{job="aws_ec2",fstype=~"ext4|xfs"} / node_filesystem_files{job="aws_ec2",fstype=~"ext4|xfs"}) * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:filesystem:used:percent
      labels:
        desc: "节点的inode的剩余可用的百分比"
        unit: "%"
        job: "aws_ec2"
#############################################################################################
#                                filefd                                                     #
    - expr: node_filefd_allocated{job="aws_ec2"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:filefd_allocated:count
      labels:
        desc: "节点的文件描述符打开个数"
        unit: "%"
        job: "aws_ec2"

- expr: node_filefd_allocated{job="aws_ec2"}/node_filefd_maximum{job="aws_ec2"} * 100 * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:filefd_allocated:percent
      labels:
        desc: "节点的文件描述符打开百分比"
        unit: "%"
        job: "aws_ec2"

#############################################################################################
#                                network                                                    #
    - expr: avg by (environment,instance,device) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netin:bit:rate
      labels:
        desc: "节点网卡eth0每秒接收的比特数"
        unit: "bit/s"
        job: "aws_ec2"

- expr: avg by (environment,instance,device) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netout:bit:rate
      labels:
        desc: "节点网卡eth0每秒发送的比特数"
        unit: "bit/s"
        job: "aws_ec2"

- expr: avg by (environment,instance,device) (irate(node_network_receive_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netin:packet:rate
      labels:
        desc: "节点网卡每秒接收的数据包个数"
        unit: "个/秒"
        job: "aws_ec2"

- expr: avg by (environment,instance,device) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netout:packet:rate
      labels:
        desc: "节点网卡发送的数据包个数"
        unit: "个/秒"
        job: "aws_ec2"

- expr: avg by (environment,instance,device) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netin:error:rate
      labels:
        desc: "节点设备驱动器检测到的接收错误包的数量"
        unit: "个/秒"
        job: "aws_ec2"

- expr: avg by (environment,instance,device) (irate(node_network_transmit_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m]))* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:netout:error:rate
      labels:
        desc: "节点设备驱动器检测到的发送错误包的数量"
        unit: "个/秒"
        job: "aws_ec2"

- expr: node_tcp_connection_states{job="aws_ec2", state="established"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:tcp:established:count
      labels:
        desc: "节点当前established的个数"
        unit: "个"
        job: "aws_ec2"

- expr: node_tcp_connection_states{job="aws_ec2", state="time_wait"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:tcp:timewait:count
      labels:
        desc: "节点timewait的连接数"
        unit: "个"
        job: "aws_ec2"

- expr: sum by (environment,instance) (node_tcp_connection_states{job="aws_ec2"})* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:network:tcp:total:count
      labels:
        desc: "节点tcp连接总数"
        unit: "个"
        job: "aws_ec2"
#############################################################################################
#                                process                                                    #
    - expr: node_processes_state{state="Z"}* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:process:zoom:total:count
      labels:
        desc: "节点当前状态为zoom的个数"
        unit: "个"
        job: "aws_ec2"
#############################################################################################
#                                other                                                    #
    - expr: abs(node_timex_offset_seconds{job="aws_ec2"})* on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:time:offset
      labels:
        desc: "节点的时间偏差"
        unit: "s"
        job: "aws_ec2"
#############################################################################################
#
    - expr: count by (instance) ( count by (instance,cpu) (node_cpu_seconds_total{ mode='system'}) ) * on(instance) group_left(nodename) (node_uname_info)
      record: node_exporter:cpu:count

```

- vim /opt/jonnyan404/prometheus/rules/node-exporter-alert.yml

```yaml
# node-exporter-alert-rules.yml
# 定义告警规则
# 通过前一个 rules 文件拿到定义的 record 别名来编写 expr 判断式
# 这里定义的告警规则，在触发的时候，都会传递到 alertmanager，最后从传递的信息中抽取所需数据发送给目标人。
groups:
  - name: node-alert
    rules:
    - alert: node-down
      expr: node_exporter:up == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "instance: {{ $labels.instance }} 宕机了"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "
        
    - alert: Prometheus无法连接Alertmanager
      expr: prometheus_notifications_alertmanagers_discovered < 1
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Prometheus not connected to alertmanager
        description: "Prometheus cannot connect the alertmanager\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

- alert: Alertmanager发送通知失败
      expr: rate(alertmanager_notifications_failed_total[1m]) > 0
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: Prometheus AlertManager notification failing
        description: "Alertmanager is failing sending notifications\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"        
        
    - alert: node-cpu-high
      expr:  node_exporter:cpu:total:percent > 80
      for: 3m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} cpu 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-cpu-iowait-high
      expr:  node_exporter:cpu:iowait:percent >= 12
      for: 3m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} cpu iowait 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-load-load1-high
      expr:  (node_exporter:load:load1) > (node_exporter:cpu:count) * 1.2
      for: 3m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} load1 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-memory-high
      expr:  node_exporter:memory:used:percent > 85
      for: 3m
      labels:
        severity: info
      annotations:
        summary: "内存使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-disk-high
      expr:  node_exporter:disk:used:percent > 80
      for: 3m
      labels:
        severity: info
      annotations:
        summary: "{{ $labels.device }}:{{ $labels.mountpoint }} 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-disk-read:count-high
      expr:  node_exporter:disk:read:count:rate > 3000
      for: 2m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} iops read 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-disk-write-count-high
      expr:  node_exporter:disk:write:count:rate > 3000
      for: 2m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} iops write 使用率高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-disk-read-mb-high
      expr:  node_exporter:disk:read:mb:rate > 60
      for: 2m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 读取字节数 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-disk-write-mb-high
      expr:  node_exporter:disk:write:mb:rate > 60
      for: 2m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 写入字节数 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-filefd-allocated-percent-high
      expr:  node_exporter:filefd_allocated:percent > 80
      for: 10m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 打开文件描述符 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-network-netin-error-rate-high
      expr:  node_exporter:network:netin:error:rate > 4
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 包进入的错误速率 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-network-netin-packet-rate-high
      expr:  node_exporter:network:netin:packet:rate > 35000
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 包进入速率 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-network-netout-packet-rate-high
      expr:  node_exporter:network:netout:packet:rate > 35000
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 包流出速率 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-network-tcp-total-count-high
      expr:  node_exporter:network:tcp:total:count > 40000
      for: 1m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} tcp连接数量 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-process-zoom-total-count-high
      expr:  node_exporter:process:zoom:total:count > 10
      for: 10m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} 僵死进程数量 高于 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: node-time-offset-high
      expr:  node_exporter:time:offset > 0.03
      for: 2m
      labels:
        severity: info
      annotations:
        summary: "instance: {{ $labels.instance }} {{ $labels.desc }}  {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

- alert: 磁盘剩余空间不足
      expr:  node_exporter:disk:used:percent > 80
      for: 2m
      labels:
        severity: warn
      annotations:
        summary: "instance: {{ $labels.instance }} 磁盘使用率已超过 {{ $value }}{{ $labels.unit }}"
        grafana: "http://x.x.x.x:3000/d/9CWBz0bik/zhu-ji-jian-kong?orgId=1&var-node={{ $labels.instance }} "

```

# 5.重启prometheus,使规则生效

```
docker restart prometheus
```