deep_scrub_choked
This commit is contained in:
a.pivkin
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_mds_check_retries }}"
|
||||
DELAY="{{ handler_health_mds_check_delay }}"
|
||||
MDS_NAME="{{ ansible_facts['hostname'] }}"
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXEC="{{ container_binary }} exec ceph-mds-{{ ansible_facts['hostname'] }}"
|
||||
{% endif %}
|
||||
|
||||
# Backward compatibility
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['fqdn'] }}.asok
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['hostname'] }}.asok
|
||||
|
||||
# First, restart the daemon
|
||||
systemctl restart ceph-mds@${MDS_NAME}
|
||||
|
||||
# Wait and ensure the socket exists after restarting the daemds
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
$DOCKER_EXEC test -S $SOCKET && exit 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# If we reach this point, it means the socket is not present.
|
||||
echo "Socket file ${SOCKET} could not be found, which means the Metadata Server is not running. Showing ceph-mds unit logs now:"
|
||||
journalctl -u ceph-mds@${MDS_NAME}
|
||||
exit 1
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_mgr_check_retries }}"
|
||||
DELAY="{{ handler_health_mgr_check_delay }}"
|
||||
MGR_NAME="{{ ansible_facts['hostname'] }}"
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXEC="{{ container_binary }} exec ceph-mgr-{{ ansible_facts['hostname'] }}"
|
||||
{% endif %}
|
||||
|
||||
# Backward compatibility
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['fqdn'] }}.asok
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['hostname'] }}.asok
|
||||
|
||||
systemctl reset-failed ceph-mgr@${MGR_NAME}
|
||||
# First, restart the daemon
|
||||
systemctl restart ceph-mgr@${MGR_NAME}
|
||||
|
||||
# Wait and ensure the socket exists after restarting the daemds
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
$DOCKER_EXEC test -S $SOCKET && exit 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# If we reach this point, it means the socket is not present.
|
||||
echo "Socket file ${SOCKET} could not be found, which means ceph manager is not running. Showing ceph-mgr unit logs now:"
|
||||
journalctl -u ceph-mgr@${MGR_NAME}
|
||||
exit 1
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_mon_check_retries }}"
|
||||
DELAY="{{ handler_health_mon_check_delay }}"
|
||||
MONITOR_NAME="{{ monitor_name }}"
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXEC="{{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }}"
|
||||
{% endif %}
|
||||
|
||||
# if daemon is uninstalled, no restarting is needed; so exit with success
|
||||
systemctl status ceph-mon@{{ ansible_facts['hostname'] }} > /dev/null
|
||||
if [[ $? -ne 0 ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Backward compatibility
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['fqdn'] }}.asok
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['hostname'] }}.asok
|
||||
|
||||
check_quorum() {
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
$DOCKER_EXEC ceph --cluster {{ cluster }} quorum_status --format json | "{{ discovered_interpreter_python }}" -c 'import sys, json; exit(0) if "{{ monitor_name }}" in json.load(sys.stdin)["quorum_names"] else exit(1)' && exit 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# If we reach this point, it means there is a problem with the quorum
|
||||
echo "Error with quorum."
|
||||
echo "cluster status:"
|
||||
$DOCKER_EXEC ceph --cluster {{ cluster }} -s
|
||||
echo "quorum status:"
|
||||
$DOCKER_EXEC ceph --cluster {{ cluster }} daemon mon.${MONITOR_NAME} mon_status
|
||||
$DOCKER_EXEC ceph --cluster {{ cluster }} daemon mon.${MONITOR_NAME} quorum_status
|
||||
exit 1
|
||||
}
|
||||
|
||||
# First, restart the daemon
|
||||
systemctl restart ceph-mon@{{ ansible_facts['hostname'] }}
|
||||
|
||||
COUNT=10
|
||||
# Wait and ensure the socket exists after restarting the daemon
|
||||
while [ $COUNT -ne 0 ]; do
|
||||
$DOCKER_EXEC test -S $SOCKET && check_quorum
|
||||
sleep $DELAY
|
||||
let COUNT=COUNT-1
|
||||
done
|
||||
# If we reach this point, it means the socket is not present.
|
||||
echo "Socket file ${SOCKET} could not be found, which means the monitor is not running. Showing ceph-mon unit logs now:"
|
||||
journalctl -u ceph-mon@{{ ansible_facts['hostname'] }}
|
||||
exit 1
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_nfs_check_retries }}"
|
||||
DELAY="{{ handler_health_nfs_check_delay }}"
|
||||
NFS_NAME="ceph-nfs@{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}"
|
||||
PID=/var/run/ganesha/ganesha.pid
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXEC="{{ container_binary }} exec ceph-nfs-{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}"
|
||||
{% endif %}
|
||||
|
||||
# First, restart the daemon
|
||||
{% if containerized_deployment | bool -%}
|
||||
systemctl restart $NFS_NAME
|
||||
# Wait and ensure the pid exists after restarting the daemon
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
$DOCKER_EXEC test -f $PID && exit 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# If we reach this point, it means the pid is not present.
|
||||
echo "PID file ${PID} could not be found, which means Ganesha is not running. Showing $NFS_NAME unit logs now:"
|
||||
journalctl -u $NFS_NAME
|
||||
exit 1
|
||||
{% else %}
|
||||
systemctl restart nfs-ganesha
|
||||
{% endif %}
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
|
||||
DELAY="{{ handler_health_osd_check_delay }}"
|
||||
CEPH_CLI="--name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring --cluster {{ cluster }}"
|
||||
|
||||
check_pgs() {
|
||||
num_pgs=$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')
|
||||
if [[ "$num_pgs" == "0" ]]; then
|
||||
return 0
|
||||
fi
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
test "$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')" -eq "$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(sum ( [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if "active+clean" in i["state_name"]]))')"
|
||||
RET=$?
|
||||
test $RET -eq 0 && return 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# PGs not clean, exiting with return code 1
|
||||
echo "Error while running 'ceph $CEPH_CLI -s', PGs were not reported as active+clean"
|
||||
echo "It is possible that the cluster has less OSDs than the replica configuration"
|
||||
echo "Will refuse to continue"
|
||||
$container_exec ceph $CEPH_CLI -s
|
||||
$container_exec ceph $CEPH_CLI osd dump
|
||||
$container_exec ceph $CEPH_CLI osd tree
|
||||
$container_exec ceph $CEPH_CLI osd crush rule dump
|
||||
exit 1
|
||||
}
|
||||
|
||||
wait_for_socket_in_container() {
|
||||
osd_mount_point=$({{ container_binary }} exec "$1" df --output=target | grep '/var/lib/ceph/osd/')
|
||||
whoami=$({{ container_binary }} exec "$1" cat $osd_mount_point/whoami)
|
||||
if ! {{ container_binary }} exec "$1" timeout 10 bash -c "while [ ! -e /var/run/ceph/{{ cluster }}-osd.${whoami}.asok ]; do sleep 1 ; done"; then
|
||||
echo "Timed out while trying to look for a Ceph OSD socket."
|
||||
echo "Abort mission!"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
get_dev_name() {
|
||||
echo $1 | sed -r 's/ceph-osd@([a-z]{1,4})\.service/\1/'
|
||||
}
|
||||
|
||||
get_container_id_from_dev_name() {
|
||||
local id
|
||||
local count
|
||||
count=10
|
||||
while [ $count -ne 0 ]; do
|
||||
id=$({{ container_binary }} ps | grep -E "${1}$" | cut -d' ' -f1)
|
||||
test "$id" != "" && break
|
||||
sleep $DELAY
|
||||
let count=count-1
|
||||
done
|
||||
echo "$id"
|
||||
}
|
||||
|
||||
# For containerized deployments, the unit file looks like: ceph-osd@sda.service
|
||||
# For non-containerized deployments, the unit file looks like: ceph-osd@NNN.service where NNN is OSD ID
|
||||
for unit in $(systemctl list-units | grep -E "loaded * active" | grep -oE "ceph-osd@([0-9]+).service"); do
|
||||
# First, restart daemon(s)
|
||||
systemctl restart "${unit}"
|
||||
# We need to wait because it may take some time for the socket to actually exists
|
||||
COUNT=10
|
||||
# Wait and ensure the socket exists after restarting the daemon
|
||||
{% if containerized_deployment | bool %}
|
||||
osd_id=$(echo ${unit#ceph-osd@} | grep -oE '[0-9]+')
|
||||
container_id=$(get_container_id_from_dev_name "ceph-osd-${osd_id}")
|
||||
container_exec="{{ container_binary }} exec $container_id"
|
||||
{% else %}
|
||||
osd_id=$(echo ${unit#ceph-osd@} | grep -oE '[0-9]+')
|
||||
{% endif %}
|
||||
SOCKET=/var/run/ceph/{{ cluster }}-osd.${osd_id}.asok
|
||||
while [ $COUNT -ne 0 ]; do
|
||||
RETRIES="{{ handler_health_osd_check_retries }}"
|
||||
$container_exec test -S "$SOCKET" && check_pgs && continue 2
|
||||
sleep $DELAY
|
||||
let COUNT=COUNT-1
|
||||
done
|
||||
# If we reach this point, it means the socket is not present.
|
||||
echo "Socket file ${SOCKET} could not be found, which means the osd daemon is not running. Showing ceph-osd unit logs now:"
|
||||
journalctl -u "${unit}"
|
||||
exit 1
|
||||
done
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_rbd_mirror_check_retries }}"
|
||||
DELAY="{{ handler_health_rbd_mirror_check_delay }}"
|
||||
RBD_MIRROR_NAME="{{ ansible_facts['hostname'] }}"
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXEC="{{ container_binary }} exec ceph-rbd-mirror-{{ ansible_facts['hostname'] }}"
|
||||
{% endif %}
|
||||
|
||||
# Backward compatibility
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['fqdn'] }}.asok
|
||||
$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['hostname'] }}.asok
|
||||
|
||||
# First, restart the daemon
|
||||
systemctl restart ceph-rbd-mirror@rbd-mirror.${RBD_MIRROR_NAME}
|
||||
|
||||
# Wait and ensure the socket exists after restarting the daemon
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
$DOCKER_EXEC test -S $SOCKET && exit 0
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
# If we reach this point, it means the socket is not present.
|
||||
echo "Socket file ${SOCKET} could not be found, which means rbd mirror is not running. Showing ceph-rbd-mirror unit logs now:"
|
||||
journalctl -u ceph-rbd-mirror@rbd-mirror.${RBD_MIRROR_NAME}
|
||||
exit 1
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/bin/bash
|
||||
|
||||
RETRIES="{{ handler_health_rgw_check_retries }}"
|
||||
DELAY="{{ handler_health_rgw_check_delay }}"
|
||||
HOST_NAME="{{ ansible_facts['hostname'] }}"
|
||||
RGW_NUMS={{ rgw_instances | length | int }}
|
||||
RGW_FRONTEND_SSL_CERT={{ radosgw_frontend_ssl_certificate }}
|
||||
if [ -n "$RGW_FRONTEND_SSL_CERT" ]; then
|
||||
RGW_PROTOCOL=https
|
||||
else
|
||||
RGW_PROTOCOL=http
|
||||
fi
|
||||
INSTANCES_NAME=({% for i in rgw_instances %}{{ i.instance_name }} {% endfor %})
|
||||
HAPROXY_BACKEND=({% for i in rgw_instances %}{{ i.haproxy_backend | default('rgw-backend') }} {% endfor %})
|
||||
RGW_IPS=({% for i in rgw_instances %}{{ i.radosgw_address }} {% endfor %})
|
||||
RGW_PORTS=({% for i in rgw_instances %}{{ i.radosgw_frontend_port }} {% endfor %})
|
||||
RGW_ZONE="{{ rgw_zone }}"
|
||||
declare -a DOCKER_EXECS
|
||||
declare -a SOCKET_PREFIX
|
||||
for ((i=0; i<${RGW_NUMS}; i++)); do
|
||||
SOCKET_PREFIX[i]="/var/run/ceph/{{ cluster }}-client.rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}"
|
||||
DOCKER_EXECS[i]=""
|
||||
{% if containerized_deployment | bool %}
|
||||
DOCKER_EXECS[i]="{{ container_binary }} exec ceph-rgw-${RGW_ZONE}-${HOST_NAME}-${INSTANCES_NAME[i]}"
|
||||
{% endif %}
|
||||
done
|
||||
|
||||
check_socket() {
|
||||
local i=$1
|
||||
local succ=0
|
||||
local count=10
|
||||
# Wait and ensure the socket exists after restarting the daemon
|
||||
while [ $count -ne 0 ]; do
|
||||
SOCKET=$(grep ${SOCKET_PREFIX[i]} /proc/net/unix | awk '{ print $8 }')
|
||||
if [ -n "${SOCKET}" ]; then
|
||||
${DOCKER_EXECS[i]} test -S ${SOCKET} && succ=$((succ+1)) && break
|
||||
fi
|
||||
sleep $DELAY
|
||||
let count=count-1
|
||||
done
|
||||
if [ $succ -ne 1 ]; then
|
||||
echo "Socket file ${SOCKET} could not be found, which means Rados Gateway is not running. Showing ceph-rgw unit logs now:"
|
||||
journalctl -u ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_for_curl_or_wget() {
|
||||
local i=$1
|
||||
url="$RGW_PROTOCOL://${RGW_IPS[i]}:${RGW_PORTS[i]}"
|
||||
if ${DOCKER_EXECS[i]} command -v wget &>/dev/null; then
|
||||
rgw_test_result=$(${DOCKER_EXECS[i]} wget --no-check-certificate --tries 1 --quiet --server-response --spider -O /dev/null 2>&1 $url | awk 'NR==1{print $2}')
|
||||
elif ${DOCKER_EXECS[i]} command -v curl &>/dev/null; then
|
||||
rgw_test_result=$(${DOCKER_EXECS[i]} curl {{ '-g' if ip_version == 'ipv6' else '' }} -k -w "%{http_code}" --silent --output /dev/null $url)
|
||||
else
|
||||
echo "It seems that neither curl nor wget are available on your system."
|
||||
echo "Cannot test rgw connection."
|
||||
rgw_test_result=0
|
||||
fi
|
||||
}
|
||||
|
||||
check_rest() {
|
||||
local i=$1
|
||||
local succ=0
|
||||
while [ $RETRIES -ne 0 ]; do
|
||||
check_for_curl_or_wget ${i}
|
||||
if [ $rgw_test_result -eq 200 ] || [ $rgw_test_result -eq 404 ] || [ $rgw_test_result -eq 405 ]; then
|
||||
succ=$((succ+1))
|
||||
break
|
||||
fi
|
||||
sleep $DELAY
|
||||
let RETRIES=RETRIES-1
|
||||
done
|
||||
if [ $succ -ne 1 ]; then
|
||||
# If we reach this point, it means there is a problem with the connection to rgw
|
||||
echo "Error connecting locally to Rados Gateway service: $RGW_PROTOCOL://${RGW_IPS[i]}:${RGW_PORTS[i]}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
for ((i=0; i<${RGW_NUMS}; i++)); do
|
||||
# Check if systemd unit exists
|
||||
# This is needed for new instances as the restart might trigger before the deployment
|
||||
if ! systemctl list-units --full --all | grep -q "ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}"; then
|
||||
echo "Systemd unit ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]} does not exist."
|
||||
continue
|
||||
fi
|
||||
|
||||
{% if handler_rgw_use_haproxy_maintenance %}
|
||||
# set server weight to 0 on haproxy
|
||||
echo "set weight ${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} 0" | socat stdio {{ haproxy_socket_path }}
|
||||
|
||||
# wait for the connections to drop
|
||||
retries={{ handler_rgw_haproxy_maintenance_retries | default(60) }}
|
||||
while [ $retries -gt 0 ]; do
|
||||
if [ "$(echo "show servers conn ${HAPROXY_BACKEND[i]}" | socat stdio {{ haproxy_socket_path }} | grep "${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} " | awk '{ print $7 }')" -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
let retries=retries-1
|
||||
done
|
||||
{% endif %}
|
||||
|
||||
# Restart the daemon
|
||||
systemctl restart ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}
|
||||
|
||||
# Check socket files
|
||||
check_socket ${i}
|
||||
# Check rest
|
||||
check_rest ${i}
|
||||
|
||||
{% if handler_rgw_use_haproxy_maintenance %}
|
||||
# set server weight to 100 on haproxy
|
||||
echo "set weight ${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} 100" | socat stdio {{ haproxy_socket_path }}
|
||||
{% endif %}
|
||||
done
|
||||
Reference in New Issue
Block a user