Prometheus Alertmanager Grafana 监控警报

Prometheus Alertmanager Grafana 监控警报

node-exporter, Linux系统信息采集组件

prometheus , 抓取、储存监控数据，供查询指标

alertmanager , 发送警报通知

grafana , web图形展示

环境 centos7+docker

单机模式，快速搭建测试环境

---

#安装node-exporter

 #docker安装方式

docker rm -f node-exporter

docker run -d -p 9100:9100 \

--name node-exporter \

-h $(hostname) \

-v "/proc:/host/proc:ro" \

-v "/sys:/host/sys:ro" \

-v "/:/rootfs:ro" \

--net="host" \

--pid="host" \

--cap-add=SYS_TIME \

  --cpus 0.1 \

  --memory 32M \

  --restart always \

prom/node-exporter \

--path.rootfs /rootfs \

--path.procfs /host/proc \

--path.sysfs /host/sys \

--collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)"

 #非docker方式，正式环境推荐

 #curl -s http://files.elven.vip/download/node_exporter.sh |bash

 #默认端口9100 可浏览器访问查看文本数据 IP:9100/metrics

---

#安装prometheus

 ##prometheus配置文件

 # mkdir -p /data/prometheus/

 # vi /data/prometheus/prometheus.yml

 #prometheus.yml

global:

  scrape_interval:     15s # 设定抓取数据的周期，默认为1min

  evaluation_interval: 15s # 设定更新rules文件的周期，默认为1min

  scrape_timeout: 15s      # 设定抓取数据的超时时间，默认为10s

 # Alertmanager配置

alerting:

  alertmanagers:

  - static_configs:

    - targets: ["localhost:9093"]

 # rule配置

rule_files:

- "/prometheus/rules.*yml"

scrape_configs:

  - job_name: 'node-exporter'

    static_configs:

      - targets: ['localhost:9100']

 ##警告规则

 # vi /data/prometheus/rules.linux.yml

 #rules

groups:

 - name: test-rules

   rules:

   - alert: InstanceDown  #告警名称

     expr: up == 0  #告警判定条件

     for: 3s  #持续多久后，才发送

     labels:  #标签

      team: test

     annotations: ##警报信息

      summary: "{{$labels.instance}}: has been down"

      description: "{{$labels.instance}}: job {{$labels.job}} has been down "

 #启动prometheus

docker rm -f prometheus

Dir=/data/prometheus

docker run -dit \

-u root --restart=always \

-h prometheus  --name prometheus \

--net="host" \

-v /etc/localtime:/etc/localtime:ro \

-v $Dir:/prometheus \

-v $Dir/prometheus.yml:/etc/prometheus/prometheus.yml \

--cpus 0.5 --memory 1024M \

prom/prometheus --web.enable-lifecycle

 #浏览器访问 IP:9090

 #修改配置后 curl -X POST http://localhost:9090/-/reload

---

#安装alertmanager

 #配置文件

 # mkdir -p /data/prometheus/alertmanager

 # vi /data/prometheus/alertmanager/alertmanager.yml

 # 全局配置项

global:

  resolve_timeout: 5m #超时,默认5min

  #邮箱smtp服务

  smtp_smarthost: 'smtp.qq.com:587'

  smtp_from: 'report@elven.vip'

  smtp_auth_username: 'report@elven.vip'

  smtp_auth_password: 'xxx密码'

  smtp_hello: 'qq.com'

 # 路由

route:

  group_by: ['alertname'] # 报警分组依据

  group_wait: 20s #组等待时间

  group_interval: 20s # 发送前等待时间

  repeat_interval: 10m #重复周期10分钟

  receiver: 'email' # 默认警报接收者

 # 警报接收者

receivers:

- name: 'email' # 警报名称

  email_configs:

  - to: '228@elven.vip'  # 接收警报的email

 #关于email、微信发送模板，后面单独讲

 #启动alertmanager

docker rm -f alertmanager

Dir=/data/prometheus/alertmanager

docker run -dit \

-u root --restart=always \

-h alertmanager  --name alertmanager \

--net="host" \

-v /etc/localtime:/etc/localtime:ro \

-v $Dir:/alertmanager \

-v $Dir/alertmanager.yml:/etc/alertmanager/alertmanager.yml \

--cpus 0.2 --memory 128M \

prom/alertmanager

 #浏览器访问 IP:9093

---

关闭node-exporter测试

docker stop node-exporter

netstat -lntp |grep 9100

查看prometheus点击Alerts查看rules生效

查看alertmanager是否收到,等待邮件通知……

---

#grafana

docker rm -f grafana

docker run -dit --name grafana \

-h grafana -u root  \

--restart always \

--cpus 0.2 --memory 128M \

-v /etc/localtime:/etc/localtime:ro \

-v /data/grafana:/var/lib/grafana \

--net="host" \

grafana/grafana

 # 浏览器访问 ip:3000 默认登录用户密码 admin

 #添加数据源Add data sources选 Prometheus 

 #可以参考这个grafana模板

 #https://grafana.com/grafana/dashboards/8919

 #导入 左边+按钮 -> Import -> 8919 -> prometheus选择源 -> Import