deep_scrub_choked
This commit is contained in:
a.pivkin
@@ -0,0 +1,115 @@
|
||||
groups:
|
||||
- name: Dashboard
|
||||
rules:
|
||||
- alert: Ceph Health Warning
|
||||
expr: ceph_health_status == 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Ceph Health Warning"
|
||||
description: "Overall Ceph Health"
|
||||
- alert: Ceph Health Error
|
||||
expr: ceph_health_status > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Ceph Health Error"
|
||||
description: "The Ceph cluster health is in an error state"
|
||||
- alert: Disk(s) Near Full
|
||||
expr: (ceph_osd_stat_bytes_used / ceph_osd_stat_bytes) * 100 > 85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Disk(s) Near Full"
|
||||
description: "This shows how many disks are at or above 85% full. Performance may degrade beyond this threshold on filestore (XFS) backed OSD's."
|
||||
- alert: OSD(s) Down
|
||||
expr: ceph_osd_up < 0.5
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "OSD(s) Down"
|
||||
description: "This indicates that one or more OSDs is currently marked down in the cluster."
|
||||
- alert: OSD Host(s) Down
|
||||
expr: count by(instance) (ceph_disk_occupation * on(ceph_daemon) group_right(instance) ceph_osd_up == 0) - count by(instance) (ceph_disk_occupation) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "OSD Host(s) Down"
|
||||
description: "This indicates that one or more OSD hosts is currently down in the cluster."
|
||||
- alert: PG(s) Stuck
|
||||
expr: max(ceph_osd_numpg) > scalar(ceph_pg_active)
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "PG(s) Stuck"
|
||||
description: "This indicates there are pg's in a stuck state, manual intervention needed to resolve."
|
||||
- alert: OSD Host Loss Check
|
||||
expr: max(sum(ceph_osd_stat_bytes - ceph_osd_stat_bytes_used)) * 0.9 < scalar(max(sum by (instance) (ceph_osd_stat_bytes + on (ceph_daemon) group_left (instance) (ceph_disk_occupation*0))))
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "OSD Host Loss Check"
|
||||
description: "This indicates that the cluster @ 90% full is not enough to support the loss of the largest OSD host."
|
||||
- alert: Slow OSD Responses
|
||||
expr: ((irate(node_disk_read_time_seconds_total[5m]) / clamp_min(irate(node_disk_reads_completed_total[5m]), 1) + irate(node_disk_write_time_seconds_total[5m]) / clamp_min(irate(node_disk_writes_completed_total[5m]), 1)) and on (instance, device) ceph_disk_occupation) > 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Slow OSD Responses"
|
||||
description: "This indicates that some OSD Latencies are above 1s."
|
||||
- alert: Network Errors
|
||||
expr: sum by (instance, device) (irate(node_network_receive_drop_total{device=~"(eth|en|bond|ib|mlx|p).*"}[5m]) + irate(node_network_receive_errs_total{device=~"(eth|en|bond|ib|mlx|p).*"}[5m]) + irate(node_network_transmit_drop_total{device=~"(eth|en|bond|ib|mlx|p).*"}[5m]) + irate(node_network_transmit_errs_total{device=~"(eth|en|bond|ib|mlx|p).*"}[5m])) > 10
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Network Errors"
|
||||
description: "This indicates that more than 10 dropped/error packets are seen in a 5m interval"
|
||||
- alert: Pool Capacity Low
|
||||
expr: (ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail) * 100 + on (pool_id) group_left (name) (ceph_pool_metadata*0)) > 85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Pool Capacity Low"
|
||||
description: "This indicates a low capacity in a pool."
|
||||
- alert: MON(s) Down
|
||||
expr: ceph_mon_quorum_status != 1
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "MON(s) down"
|
||||
description: "This indicates that one or more MON(s) is down."
|
||||
- alert: Cluster Capacity Low
|
||||
expr: sum(ceph_osd_stat_bytes_used) / sum(ceph_osd_stat_bytes) > 0.85
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Cluster Capacity Low"
|
||||
description: "This indicates raw used space crosses the 85% capacity threshold of the ceph cluster."
|
||||
- alert: OSD(s) with High PG Count
|
||||
expr: ceph_osd_numpg > 275
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "OSD(s) with High PG Count"
|
||||
description: "This indicates there are some OSDs with high PG count (275+)."
|
||||
- alert: Slow OSD Ops
|
||||
expr: ceph_healthcheck_slow_ops > 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: "Slow OSD Ops"
|
||||
description: "OSD requests are taking too long to process (osd_op_complaint_time exceeded)"
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Service handler
|
||||
# We use the systemd module here so we can use the daemon_reload feature,
|
||||
# since we're shipping the .service file ourselves
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ item }}"
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: restarted
|
||||
with_items:
|
||||
- 'alertmanager'
|
||||
- 'prometheus'
|
||||
when: not docker2podman | default(False) | bool
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
galaxy_info:
|
||||
company: Red Hat
|
||||
author: Boris Ranto
|
||||
description: Configures Prometheus for Ceph Dashboard
|
||||
license: Apache
|
||||
min_ansible_version: '2.4'
|
||||
platforms:
|
||||
- name: EL
|
||||
versions:
|
||||
- 'all'
|
||||
galaxy_tags:
|
||||
- system
|
||||
dependencies: []
|
||||
@@ -0,0 +1,63 @@
|
||||
---
|
||||
- name: Create prometheus directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0755"
|
||||
with_items:
|
||||
- "{{ prometheus_conf_dir }}"
|
||||
- "{{ prometheus_data_dir }}"
|
||||
|
||||
- name: Write prometheus config file
|
||||
openstack.config_template.config_template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ prometheus_conf_dir }}/prometheus.yml"
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0640"
|
||||
config_type: yaml
|
||||
config_overrides: "{{ prometheus_conf_overrides }}"
|
||||
notify: Service handler
|
||||
|
||||
- name: Make sure the alerting rules directory exists
|
||||
ansible.builtin.file:
|
||||
path: "/etc/prometheus/alerting/"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0755"
|
||||
|
||||
- name: Copy alerting rules
|
||||
ansible.builtin.copy:
|
||||
src: "ceph_dashboard.yml"
|
||||
dest: "/etc/prometheus/alerting/ceph_dashboard.yml"
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0644"
|
||||
|
||||
- name: Create alertmanager directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0755"
|
||||
with_items:
|
||||
- "{{ alertmanager_conf_dir }}"
|
||||
- "{{ alertmanager_data_dir }}"
|
||||
|
||||
- name: Write alertmanager config file
|
||||
openstack.config_template.config_template:
|
||||
src: alertmanager.yml.j2
|
||||
dest: "{{ alertmanager_conf_dir }}/alertmanager.yml"
|
||||
owner: "{{ prometheus_user_id }}"
|
||||
group: "{{ prometheus_user_id }}"
|
||||
mode: "0640"
|
||||
config_type: yaml
|
||||
config_overrides: "{{ alertmanager_conf_overrides }}"
|
||||
notify: Service handler
|
||||
|
||||
- name: Include setup_container.yml
|
||||
ansible.builtin.include_tasks: setup_container.yml
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Include_tasks systemd.yml
|
||||
ansible.builtin.include_tasks: systemd.yml
|
||||
|
||||
- name: Start prometheus services
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ item }}"
|
||||
daemon_reload: true
|
||||
enabled: true
|
||||
state: started
|
||||
with_items:
|
||||
- prometheus
|
||||
- alertmanager
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
- name: Ship systemd services
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
with_items:
|
||||
- 'alertmanager.service'
|
||||
- 'prometheus.service'
|
||||
notify: Service handler
|
||||
@@ -0,0 +1,58 @@
|
||||
# This file is managed by ansible, don't make changes here - they will be
|
||||
# overwritten.
|
||||
[Unit]
|
||||
Description=alertmanager
|
||||
{% if container_binary == 'docker' %}
|
||||
After=docker.service network-online.target local-fs.target time-sync.target
|
||||
Requires=docker.service
|
||||
{% else %}
|
||||
After=network-online.target local-fs.target time-sync.target
|
||||
{% endif %}
|
||||
Wants=network-online.target local-fs.target time-sync.target
|
||||
|
||||
[Service]
|
||||
WorkingDirectory={{ alertmanager_data_dir }}
|
||||
EnvironmentFile=-/etc/environment
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid
|
||||
ExecStartPre=-/usr/bin/{{ container_binary }} rm --storage alertmanager
|
||||
{% endif %}
|
||||
ExecStartPre=-/usr/bin/{{ container_binary }} rm -f alertmanager
|
||||
ExecStart=/usr/bin/{{ container_binary }} run --rm --name=alertmanager \
|
||||
{% if container_binary == 'podman' %}
|
||||
-d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \
|
||||
{% endif %}
|
||||
--pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
|
||||
-v "{{ alertmanager_conf_dir }}:/etc/alertmanager:Z" \
|
||||
-v "{{ alertmanager_data_dir }}:/alertmanager:Z" \
|
||||
--net=host \
|
||||
--cpu-period={{ alertmanager_container_cpu_period }} \
|
||||
--cpu-quota={{ alertmanager_container_cpu_period * alertmanager_container_cpu_cores }} \
|
||||
--memory={{ alertmanager_container_memory }}GB \
|
||||
--memory-swap={{ alertmanager_container_memory * 2 }}GB \
|
||||
{{ alertmanager_container_image }} \
|
||||
--config.file=/etc/alertmanager/alertmanager.yml \
|
||||
--cluster.listen-address={{ grafana_server_addr }}:{{ alertmanager_cluster_port }} \
|
||||
{% for peer in grafana_server_addrs|difference(grafana_server_addr) %}
|
||||
--cluster.peer={{ peer }}:{{ alertmanager_cluster_port }} \
|
||||
{% endfor %}
|
||||
--storage.path=/alertmanager \
|
||||
--web.external-url=http://{{ ansible_facts['fqdn'] }}:{{ alertmanager_port }}/ \
|
||||
--web.listen-address={{ grafana_server_addr }}:{{ alertmanager_port }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
{% else %}
|
||||
ExecStop=/usr/bin/{{ container_binary }} stop alertmanager
|
||||
{% endif %}
|
||||
KillMode=none
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
TimeoutStartSec=120
|
||||
TimeoutStopSec=15
|
||||
{% if container_binary == 'podman' %}
|
||||
Type=forking
|
||||
PIDFile=/%t/%n-pid
|
||||
{% endif %}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,20 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
group_by: ['alertname']
|
||||
group_wait: 10s
|
||||
group_interval: 10s
|
||||
repeat_interval: 1h
|
||||
receiver: 'ceph-dashboard'
|
||||
receivers:
|
||||
- name: 'ceph-dashboard'
|
||||
webhook_configs:
|
||||
{% for host in groups['mgrs'] | default(groups['mons']) %}
|
||||
- url: '{{ dashboard_protocol }}://{{ hostvars[host]['ansible_facts']['fqdn'] }}:{{ dashboard_port }}/api/prometheus_receiver'
|
||||
{% if dashboard_protocol == 'https' and alertmanager_dashboard_api_no_ssl_verify | bool %}
|
||||
http_config:
|
||||
tls_config:
|
||||
insecure_skip_verify: true
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,57 @@
|
||||
# This file is managed by ansible, don't make changes here - they will be
|
||||
# overwritten.
|
||||
[Unit]
|
||||
Description=prometheus
|
||||
{% if container_binary == 'docker' %}
|
||||
After=docker.service network-online.target local-fs.target time-sync.target
|
||||
Requires=docker.service
|
||||
{% else %}
|
||||
After=network-online.target local-fs.target time-sync.target
|
||||
{% endif %}
|
||||
Wants=network-online.target local-fs.target time-sync.target
|
||||
|
||||
[Service]
|
||||
EnvironmentFile=-/etc/environment
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStartPre=-/usr/bin/rm -f /%t/%n-pid /%t/%n-cid
|
||||
ExecStartPre=-/usr/bin/{{ container_binary }} rm --storage prometheus
|
||||
{% endif %}
|
||||
ExecStartPre=-/usr/bin/{{ container_binary }} rm -f prometheus
|
||||
ExecStart=/usr/bin/{{ container_binary }} run --rm --name=prometheus \
|
||||
{% if container_binary == 'podman' %}
|
||||
-d --log-driver journald --conmon-pidfile /%t/%n-pid --cidfile /%t/%n-cid \
|
||||
{% endif %}
|
||||
--pids-limit={{ 0 if container_binary == 'podman' else -1 }} \
|
||||
-v "{{ prometheus_conf_dir }}:/etc/prometheus:Z" \
|
||||
-v "{{ prometheus_data_dir }}:/prometheus:Z" \
|
||||
--net=host \
|
||||
--user={{ prometheus_user_id }} \
|
||||
--cpu-period={{ prometheus_container_cpu_period }} \
|
||||
--cpu-quota={{ prometheus_container_cpu_period * prometheus_container_cpu_cores }} \
|
||||
--memory={{ prometheus_container_memory }}GB \
|
||||
--memory-swap={{ prometheus_container_memory * 2 }}GB \
|
||||
{{ prometheus_container_image }} \
|
||||
--config.file=/etc/prometheus/prometheus.yml \
|
||||
--storage.tsdb.path=/prometheus \
|
||||
{% if prometheus_storage_tsdb_retention_time is defined %}
|
||||
--storage.tsdb.retention.time={{ prometheus_storage_tsdb_retention_time }} \
|
||||
{% endif %}
|
||||
--web.external-url=http://{{ ansible_facts['fqdn'] }}:{{ prometheus_port }}/ \
|
||||
--web.listen-address={{ grafana_server_addr }}:{{ prometheus_port }}
|
||||
{% if container_binary == 'podman' %}
|
||||
ExecStop=/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`"
|
||||
{% else %}
|
||||
ExecStop=/usr/bin/{{ container_binary }} stop prometheus
|
||||
{% endif %}
|
||||
KillMode=none
|
||||
Restart=always
|
||||
RestartSec=10s
|
||||
TimeoutStartSec=120
|
||||
TimeoutStopSec=15
|
||||
{% if container_binary == 'podman' %}
|
||||
Type=forking
|
||||
PIDFile=/%t/%n-pid
|
||||
{% endif %}
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,38 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
rule_files:
|
||||
- '/etc/prometheus/alerting/*'
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['{{ grafana_server_addr }}:{{ prometheus_port }}']
|
||||
- job_name: 'ceph'
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
{% for host in groups[mgr_group_name] | default(groups[mon_group_name]) %}
|
||||
- targets: ['{{ host }}:9283']
|
||||
labels:
|
||||
instance: 'ceph_cluster'
|
||||
{% endfor %}
|
||||
- job_name: 'node'
|
||||
static_configs:
|
||||
{% for host in (groups['all'] | difference(groups[monitoring_group_name] | union(groups.get(client_group_name, []))) | union(groups.get(osd_group_name, []))) %}
|
||||
- targets: ['{{ host }}:{{ node_exporter_port }}']
|
||||
labels:
|
||||
instance: "{{ hostvars[host]['ansible_facts']['nodename'] }}"
|
||||
{% endfor %}
|
||||
- job_name: 'grafana'
|
||||
static_configs:
|
||||
{% for host in groups[monitoring_group_name] %}
|
||||
- targets: ['{{ host }}:{{ node_exporter_port }}']
|
||||
labels:
|
||||
instance: "{{ hostvars[host]['ansible_facts']['nodename'] }}"
|
||||
{% endfor %}
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- scheme: http
|
||||
static_configs:
|
||||
- targets: ['{{ grafana_server_addr }}:{{ alertmanager_port }}']
|
||||
Reference in New Issue
Block a user