deep_scrub_choked

roles/ceph-handler/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!) The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2014] [Guillaume Abrioux]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

roles/ceph-handler/README.md

@@ -0,0 +1,2 @@
+# Ansible role: ceph-handler
+Documentation is available at http://docs.ceph.com/ceph-ansible/.

roles/ceph-handler/handlers/main.yml

@@ -0,0 +1,77 @@
+---
+- name: Handlers
+  when:
+    - not rolling_update | bool
+    - not docker2podman | default(False) | bool
+  block:
+    - name: Make tempdir for scripts
+      ansible.builtin.tempfile:
+        state: directory
+        prefix: ceph_ansible
+      listen:
+        - "Restart ceph mons"
+        - "Restart ceph osds"
+        - "Restart ceph mdss"
+        - "Restart ceph rgws"
+        - "Restart ceph nfss"
+        - "Restart ceph rbdmirrors"
+        - "Restart ceph mgrs"
+      register: tmpdirpath
+      when: tmpdirpath is not defined or tmpdirpath.path is not defined or tmpdirpath.state=="absent"
+
+    - name: Mons handler
+      ansible.builtin.include_tasks: handler_mons.yml
+      when: mon_group_name in group_names
+      listen: "Restart ceph mons"
+
+    - name: Osds handler
+      ansible.builtin.include_tasks: handler_osds.yml
+      when: osd_group_name in group_names
+      listen: "Restart ceph osds"
+
+    - name: Mdss handler
+      ansible.builtin.include_tasks: handler_mdss.yml
+      when: mds_group_name in group_names
+      listen: "Restart ceph mdss"
+
+    - name: Rgws handler
+      ansible.builtin.include_tasks: handler_rgws.yml
+      when: rgw_group_name in group_names
+      listen: "Restart ceph rgws"
+
+    - name: Nfss handler
+      ansible.builtin.include_tasks: handler_nfss.yml
+      when: nfs_group_name in group_names
+      listen: "Restart ceph nfss"
+
+    - name: Rbdmirrors handler
+      ansible.builtin.include_tasks: handler_rbdmirrors.yml
+      when: rbdmirror_group_name in group_names
+      listen: "Restart ceph rbdmirrors"
+
+    - name: Mgrs handler
+      ansible.builtin.include_tasks: handler_mgrs.yml
+      when: mgr_group_name in group_names
+      listen: "Restart ceph mgrs"
+
+    - name: Ceph crash handler
+      ansible.builtin.include_tasks: handler_crash.yml
+      listen: "Restart ceph crash"
+      when:
+        - inventory_hostname in groups.get(mon_group_name, [])
+          or inventory_hostname in groups.get(mgr_group_name, [])
+          or inventory_hostname in groups.get(osd_group_name, [])
+          or inventory_hostname in groups.get(mds_group_name, [])
+          or inventory_hostname in groups.get(rgw_group_name, [])
+          or inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+    - name: Ceph exporter handler
+      ansible.builtin.include_tasks: handler_exporter.yml
+      listen: "Restart ceph exporter"
+      when:
+        - inventory_hostname in groups.get(mon_group_name, [])
+          or inventory_hostname in groups.get(mgr_group_name, [])
+          or inventory_hostname in groups.get(osd_group_name, [])
+          or inventory_hostname in groups.get(mds_group_name, [])
+          or inventory_hostname in groups.get(rgw_group_name, [])
+          or inventory_hostname in groups.get(rbdmirror_group_name, [])

roles/ceph-handler/meta/main.yml

@@ -0,0 +1,14 @@
+---
+galaxy_info:
+  company: Red Hat
+  author: Sébastien Han
+  description: Contains handlers for Ceph services
+  license: Apache
+  min_ansible_version: '2.7'
+  platforms:
+    - name: EL
+      versions:
+        - 'all'
+  galaxy_tags:
+    - system
+dependencies: []

roles/ceph-handler/tasks/check_running_cluster.yml

@@ -0,0 +1,8 @@
+---
+- name: Include check_running_containers.yml
+  ansible.builtin.include_tasks: check_running_containers.yml
+  when: containerized_deployment | bool
+
+- name: Include check_socket_non_container.yml
+  ansible.builtin.include_tasks: check_socket_non_container.yml
+  when: not containerized_deployment | bool

roles/ceph-handler/tasks/check_running_containers.yml

@@ -0,0 +1,84 @@
+---
+- name: Check for a mon container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-mon-{{ ansible_facts['hostname'] }}'"
+  register: ceph_mon_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(mon_group_name, [])
+
+- name: Check for an osd container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-osd'"
+  register: ceph_osd_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(osd_group_name, [])
+
+- name: Check for a mds container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-mds-{{ ansible_facts['hostname'] }}'"
+  register: ceph_mds_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(mds_group_name, [])
+
+- name: Check for a rgw container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-rgw-{{ rgw_zone }}-{{ ansible_facts['hostname'] }}'"
+  register: ceph_rgw_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(rgw_group_name, [])
+
+- name: Check for a mgr container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-mgr-{{ ansible_facts['hostname'] }}'"
+  register: ceph_mgr_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(mgr_group_name, [])
+
+- name: Check for a rbd mirror container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-rbd-mirror-{{ ansible_facts['hostname'] }}'"
+  register: ceph_rbd_mirror_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Check for a nfs container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-nfs-{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}'"
+  register: ceph_nfs_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(nfs_group_name, [])
+
+- name: Check for a ceph-crash container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-crash-{{ ansible_facts['hostname'] }}'"
+  register: ceph_crash_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Check for a ceph-exporter container
+  ansible.builtin.command: "{{ container_binary }} ps -q --filter='name=ceph-exporter-{{ ansible_facts['hostname'] }}'"
+  register: ceph_exporter_container_stat
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])

roles/ceph-handler/tasks/check_socket_non_container.yml

@@ -0,0 +1,234 @@
+---
+- name: Find ceph mon socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-mon*.asok"
+    use_regex: false
+  register: mon_socket_stat
+  when: inventory_hostname in groups.get(mon_group_name, [])
+
+- name: Check if the ceph mon socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: mon_socket
+  with_items: "{{ mon_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+    - mon_socket_stat.files | length > 0
+
+- name: Remove ceph mon socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ mon_socket_stat.files }}"
+    - "{{ mon_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+    - mon_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Find ceph osd socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-osd.*.asok"
+    use_regex: false
+  register: osd_socket_stat
+  when: inventory_hostname in groups.get(osd_group_name, [])
+
+- name: Check if the ceph osd socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: osd_socket
+  with_items: "{{ osd_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(osd_group_name, [])
+    - osd_socket_stat.files | length > 0
+
+- name: Remove ceph osd socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ osd_socket_stat.files }}"
+    - "{{ osd_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(osd_group_name, [])
+    - osd_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Find ceph osd socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-mds*.asok"
+    use_regex: false
+  register: mds_socket_stat
+  when: inventory_hostname in groups.get(mds_group_name, [])
+
+- name: Check if the ceph mds socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: mds_socket
+  with_items: "{{ mds_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(mds_group_name, [])
+    - mds_socket_stat.files | length > 0
+
+- name: Remove ceph mds socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ mds_socket_stat.files }}"
+    - "{{ mds_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(mds_group_name, [])
+    - mds_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Find ceph rgw socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-client.rgw*.asok"
+    use_regex: false
+  register: rgw_socket_stat
+  when: inventory_hostname in groups.get(rgw_group_name, [])
+
+- name: Check if the ceph rgw socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: rgw_socket
+  with_items: "{{ rgw_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(rgw_group_name, [])
+    - rgw_socket_stat.files | length > 0
+
+- name: Remove ceph rgw socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ rgw_socket_stat.files }}"
+    - "{{ rgw_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(rgw_group_name, [])
+    - rgw_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Find ceph mgr socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-mgr*.asok"
+    use_regex: false
+  register: mgr_socket_stat
+  when: inventory_hostname in groups.get(mgr_group_name, [])
+
+- name: Check if the ceph mgr socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: mgr_socket
+  with_items: "{{ mgr_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(mgr_group_name, [])
+    - mgr_socket_stat.files | length > 0
+
+- name: Remove ceph mgr socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ mgr_socket_stat.files }}"
+    - "{{ mgr_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(mgr_group_name, [])
+    - mgr_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Find ceph rbd mirror socket
+  ansible.builtin.find:
+    paths: ["{{ rbd_client_admin_socket_path }}"]
+    recurse: true
+    file_type: any
+    patterns: "{{ cluster }}-client.rbd-mirror*.asok"
+    use_regex: false
+  register: rbd_mirror_socket_stat
+  when: inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Check if the ceph rbd mirror socket is in-use
+  ansible.builtin.command: grep -q {{ item.path }} /proc/net/unix
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: rbd_mirror_socket
+  with_items: "{{ rbd_mirror_socket_stat.files }}"
+  when:
+    - inventory_hostname in groups.get(rbdmirror_group_name, [])
+    - rbd_mirror_socket_stat.files | length > 0
+
+- name: Remove ceph rbd mirror socket if exists and not used by a process
+  ansible.builtin.file:
+    name: "{{ item.0.path }}"
+    state: absent
+  with_together:
+    - "{{ rbd_mirror_socket_stat.files }}"
+    - "{{ rbd_mirror_socket.results }}"
+  when:
+    - inventory_hostname in groups.get(rbdmirror_group_name, [])
+    - rbd_mirror_socket_stat.files | length > 0
+    - item.1.rc == 1
+
+- name: Check for a nfs ganesha pid
+  ansible.builtin.command: "pgrep ganesha.nfsd"
+  register: nfs_process
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  when: inventory_hostname in groups.get(nfs_group_name, [])
+
+- name: Check for a ceph-crash process
+  ansible.builtin.command: pgrep ceph-crash
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: crash_process
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Check for a ceph-exporter process
+  ansible.builtin.command: pgrep ceph-exporter
+  changed_when: false
+  failed_when: false
+  check_mode: false
+  register: exporter_process
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])

roles/ceph-handler/tasks/handler_crash.yml

@@ -0,0 +1,18 @@
+---
+- name: Set _crash_handler_called before restart
+  ansible.builtin.set_fact:
+    _crash_handler_called: true
+
+- name: Restart the ceph-crash service  # noqa: ignore-errors
+  ansible.builtin.systemd:
+    name: ceph-crash@{{ ansible_facts['hostname'] }}
+    state: restarted
+    enabled: true
+    masked: false
+    daemon_reload: true
+  ignore_errors: true
+  when: hostvars[inventory_hostname]['_crash_handler_called'] | default(False) | bool
+
+- name: Set _crash_handler_called after restart
+  ansible.builtin.set_fact:
+    _crash_handler_called: false

roles/ceph-handler/tasks/handler_exporter.yml

@@ -0,0 +1,18 @@
+---
+- name: Set _exporter_handler_called before restart
+  ansible.builtin.set_fact:
+    _exporter_handler_called: true
+
+- name: Restart the ceph-exporter service  # noqa: ignore-errors
+  ansible.builtin.systemd:
+    name: ceph-exporter@{{ ansible_facts['hostname'] }}
+    state: restarted
+    enabled: true
+    masked: false
+    daemon_reload: true
+  ignore_errors: true
+  when: hostvars[inventory_hostname]['_exporter_handler_called'] | default(False) | bool
+
+- name: Set _exporter_handler_called after restart
+  ansible.builtin.set_fact:
+    _exporter_handler_called: false

roles/ceph-handler/tasks/handler_mdss.yml

@@ -0,0 +1,28 @@
+---
+- name: Set _mds_handler_called before restart
+  ansible.builtin.set_fact:
+    _mds_handler_called: true
+
+- name: Copy mds restart script
+  ansible.builtin.template:
+    src: restart_mds_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_mds_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph mds daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_mds_daemon.sh
+  when:
+    - hostvars[item]['handler_mds_status'] | default(False) | bool
+    - hostvars[item]['_mds_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[mds_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _mds_handler_called after restart
+  ansible.builtin.set_fact:
+    _mds_handler_called: false

roles/ceph-handler/tasks/handler_mgrs.yml

@@ -0,0 +1,28 @@
+---
+- name: Set _mgr_handler_called before restart
+  ansible.builtin.set_fact:
+    _mgr_handler_called: true
+
+- name: Copy mgr restart script
+  ansible.builtin.template:
+    src: restart_mgr_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_mgr_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph mgr daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_mgr_daemon.sh
+  when:
+    - hostvars[item]['handler_mgr_status'] | default(False) | bool
+    - hostvars[item]['_mgr_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[mgr_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _mgr_handler_called after restart
+  ansible.builtin.set_fact:
+    _mgr_handler_called: false

roles/ceph-handler/tasks/handler_mons.yml

@@ -0,0 +1,32 @@
+---
+# We only want to restart on hosts that have called the handler.
+# This var is set when he handler is called, and unset after the
+# restart to ensure only the correct hosts are restarted.
+- name: Set _mon_handler_called before restart
+  ansible.builtin.set_fact:
+    _mon_handler_called: true
+
+- name: Copy mon restart script
+  ansible.builtin.template:
+    src: restart_mon_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_mon_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph mon daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_mon_daemon.sh
+  when:
+    # We do not want to run these checks on initial deployment (`socket.rc == 0`)
+    - hostvars[item]['handler_mon_status'] | default(False) | bool
+    - hostvars[item]['_mon_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[mon_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _mon_handler_called after restart
+  ansible.builtin.set_fact:
+    _mon_handler_called: false

roles/ceph-handler/tasks/handler_nfss.yml

@@ -0,0 +1,28 @@
+---
+- name: Set _nfs_handler_called before restart
+  ansible.builtin.set_fact:
+    _nfs_handler_called: true
+
+- name: Copy nfs restart script
+  ansible.builtin.template:
+    src: restart_nfs_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_nfs_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph nfs daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_nfs_daemon.sh
+  when:
+    - hostvars[item]['handler_nfs_status'] | default(False) | bool
+    - hostvars[item]['_nfs_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[nfs_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _nfs_handler_called after restart
+  ansible.builtin.set_fact:
+    _nfs_handler_called: false

roles/ceph-handler/tasks/handler_osds.yml

@@ -0,0 +1,122 @@
+---
+- name: Set_fact trigger_restart
+  ansible.builtin.set_fact:
+    trigger_restart: true
+  loop: "{{ groups[osd_group_name] }}"
+  when: hostvars[item]['handler_osd_status'] | default(False) | bool
+  run_once: true
+
+- name: Osd handler
+  when: trigger_restart | default(False) | bool
+  block:
+    - name: Set _osd_handler_called before restart
+      ansible.builtin.set_fact:
+        _osd_handler_called: true
+
+    - name: Unset noup flag
+      ceph_osd_flag:
+        name: noup
+        cluster: "{{ cluster }}"
+        state: absent
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      run_once: true
+
+    # This does not just restart OSDs but everything else too. Unfortunately
+    # at this time the ansible role does not have an OSD id list to use
+    # for restarting them specifically.
+    # This does not need to run during a rolling update as the playbook will
+    # restart all OSDs using the tasks "start ceph osd" or
+    # "restart containerized ceph osd"
+    - name: Copy osd restart script
+      ansible.builtin.template:
+        src: restart_osd_daemon.sh.j2
+        dest: "{{ tmpdirpath.path }}/restart_osd_daemon.sh"
+        owner: root
+        group: root
+        mode: "0750"
+      when: tmpdirpath.path is defined
+
+    - name: Get pool list
+      ansible.builtin.command: "{{ ceph_cmd }} --cluster {{ cluster }} osd pool ls detail -f json"
+      register: pool_list
+      delegate_to: "{{ groups.get(mon_group_name, [])[0] }}"
+      run_once: true
+      changed_when: false
+      check_mode: false
+
+    - name: Get balancer module status
+      ansible.builtin.command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer status -f json"
+      register: balancer_status
+      run_once: true
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      changed_when: false
+      check_mode: false
+
+    - name: Set_fact pools_pgautoscaler_mode
+      ansible.builtin.set_fact:
+        pools_pgautoscaler_mode: "{{ pools_pgautoscaler_mode | default([]) | union([{'name': item.pool_name, 'mode': item.pg_autoscale_mode}]) }}"
+      run_once: true
+      with_items: "{{ pool_list.stdout | default('{}') | from_json }}"
+
+    - name: Disable balancer
+      ansible.builtin.command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer off"
+      run_once: true
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      changed_when: false
+      when: (balancer_status.stdout | from_json)['active'] | bool
+
+    - name: Disable pg autoscale on pools
+      ceph_pool:
+        name: "{{ item.name }}"
+        cluster: "{{ cluster }}"
+        pg_autoscale_mode: false
+      with_items: "{{ pools_pgautoscaler_mode }}"
+      delegate_to: "{{ groups.get(mon_group_name, [])[0] }}"
+      run_once: true
+      when:
+        - pools_pgautoscaler_mode is defined
+        - item.mode == 'on'
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+
+    - name: Restart ceph osds daemon(s)
+      ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_osd_daemon.sh
+      when:
+        - hostvars[item]['handler_osd_status'] | default(False) | bool
+        - handler_health_osd_check | bool
+        - hostvars[item]['_osd_handler_called'] | default(False) | bool
+        - hostvars[item].tmpdirpath.path is defined
+      with_items: "{{ groups[osd_group_name] | intersect(ansible_play_batch) }}"
+      delegate_to: "{{ item }}"
+      changed_when: false
+      run_once: true
+
+    - name: Set _osd_handler_called after restart
+      ansible.builtin.set_fact:
+        _osd_handler_called: false
+
+    - name: Re-enable pg autoscale on pools
+      ceph_pool:
+        name: "{{ item.name }}"
+        cluster: "{{ cluster }}"
+        pg_autoscale_mode: true
+      with_items: "{{ pools_pgautoscaler_mode }}"
+      run_once: true
+      delegate_to: "{{ groups.get(mon_group_name, [])[0] }}"
+      when:
+        - pools_pgautoscaler_mode is defined
+        - item.mode == 'on'
+      environment:
+        CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+        CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+
+    - name: Re-enable balancer
+      ansible.builtin.command: "{{ ceph_cmd }} --cluster {{ cluster }} balancer on"
+      run_once: true
+      delegate_to: "{{ groups[mon_group_name][0] }}"
+      changed_when: false
+      when: (balancer_status.stdout | from_json)['active'] | bool

roles/ceph-handler/tasks/handler_rbdmirrors.yml

@@ -0,0 +1,28 @@
+---
+- name: Set _rbdmirror_handler_called before restart
+  ansible.builtin.set_fact:
+    _rbdmirror_handler_called: true
+
+- name: Copy rbd mirror restart script
+  ansible.builtin.template:
+    src: restart_rbd_mirror_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_rbd_mirror_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph rbd mirror daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_rbd_mirror_daemon.sh
+  when:
+    - hostvars[item]['handler_rbd_mirror_status'] | default(False) | bool
+    - hostvars[item]['_rbdmirror_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[rbdmirror_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _rbdmirror_handler_called after restart
+  ansible.builtin.set_fact:
+    _rbdmirror_handler_called: false

roles/ceph-handler/tasks/handler_rgws.yml

@@ -0,0 +1,28 @@
+---
+- name: Set _rgw_handler_called before restart
+  ansible.builtin.set_fact:
+    _rgw_handler_called: true
+
+- name: Copy rgw restart script
+  ansible.builtin.template:
+    src: restart_rgw_daemon.sh.j2
+    dest: "{{ tmpdirpath.path }}/restart_rgw_daemon.sh"
+    owner: root
+    group: root
+    mode: "0750"
+  when: tmpdirpath.path is defined
+
+- name: Restart ceph rgw daemon(s)
+  ansible.builtin.command: /usr/bin/env bash {{ hostvars[item]['tmpdirpath']['path'] }}/restart_rgw_daemon.sh
+  when:
+    - hostvars[item]['handler_rgw_status'] | default(False) | bool
+    - hostvars[item]['_rgw_handler_called'] | default(False) | bool
+    - hostvars[item].tmpdirpath.path is defined
+  with_items: "{{ groups[rgw_group_name] }}"
+  delegate_to: "{{ item }}"
+  changed_when: false
+  run_once: true
+
+- name: Set _rgw_handler_called after restart
+  ansible.builtin.set_fact:
+    _rgw_handler_called: false

roles/ceph-handler/tasks/main.yml

@@ -0,0 +1,61 @@
+---
+- name: Include check_running_cluster.yml
+  ansible.builtin.include_tasks: check_running_cluster.yml
+
+# We do not want to run these checks on initial deployment (`socket.rc == 0`)
+- name: Set_fact handler_mon_status
+  ansible.builtin.set_fact:
+    handler_mon_status: "{{ 0 in (mon_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_mon_container_stat.get('rc') == 0 and ceph_mon_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(mon_group_name, [])
+
+- name: Set_fact handler_osd_status
+  ansible.builtin.set_fact:
+    handler_osd_status: "{{ 0 in (osd_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_osd_container_stat.get('rc') == 0 and ceph_osd_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(osd_group_name, [])
+
+- name: Set_fact handler_mds_status
+  ansible.builtin.set_fact:
+    handler_mds_status: "{{ 0 in (mds_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_mds_container_stat.get('rc') == 0 and ceph_mds_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(mds_group_name, [])
+
+- name: Set_fact handler_rgw_status
+  ansible.builtin.set_fact:
+    handler_rgw_status: "{{ 0 in (rgw_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_rgw_container_stat.get('rc') == 0 and ceph_rgw_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(rgw_group_name, [])
+
+- name: Set_fact handler_nfs_status
+  ansible.builtin.set_fact:
+    handler_nfs_status: "{{ (nfs_process.get('rc') == 0) if not containerized_deployment | bool else (ceph_nfs_container_stat.get('rc') == 0 and ceph_nfs_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(nfs_group_name, [])
+
+- name: Set_fact handler_rbd_status
+  ansible.builtin.set_fact:
+    handler_rbd_mirror_status: "{{ 0 in (rbd_mirror_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_rbd_mirror_container_stat.get('rc') == 0 and ceph_rbd_mirror_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Set_fact handler_mgr_status
+  ansible.builtin.set_fact:
+    handler_mgr_status: "{{ 0 in (mgr_socket.results | map(attribute='rc') | list) if not containerized_deployment | bool else (ceph_mgr_container_stat.get('rc') == 0 and ceph_mgr_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when: inventory_hostname in groups.get(mgr_group_name, [])
+
+- name: Set_fact handler_crash_status
+  ansible.builtin.set_fact:
+    handler_crash_status: "{{ crash_process.get('rc') == 0 if not containerized_deployment | bool else (ceph_crash_container_stat.get('rc') == 0 and ceph_crash_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])
+
+- name: Set_fact handler_exporter_status
+  ansible.builtin.set_fact:
+    handler_exporter_status: "{{ exporter_process.get('rc') == 0 if not containerized_deployment | bool else (ceph_exporter_container_stat.get('rc') == 0 and ceph_exporter_container_stat.get('stdout_lines', []) | length != 0) }}"
+  when:
+    - inventory_hostname in groups.get(mon_group_name, [])
+      or inventory_hostname in groups.get(mgr_group_name, [])
+      or inventory_hostname in groups.get(osd_group_name, [])
+      or inventory_hostname in groups.get(mds_group_name, [])
+      or inventory_hostname in groups.get(rgw_group_name, [])
+      or inventory_hostname in groups.get(rbdmirror_group_name, [])

roles/ceph-handler/templates/restart_mds_daemon.sh.j2

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_mds_check_retries }}"
+DELAY="{{ handler_health_mds_check_delay }}"
+MDS_NAME="{{ ansible_facts['hostname'] }}"
+{% if containerized_deployment | bool %}
+DOCKER_EXEC="{{ container_binary }} exec ceph-mds-{{ ansible_facts['hostname'] }}"
+{% endif %}
+
+# Backward compatibility
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['fqdn'] }}.asok
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mds.{{ ansible_facts['hostname'] }}.asok
+
+# First, restart the daemon
+systemctl restart ceph-mds@${MDS_NAME}
+
+# Wait and ensure the socket exists after restarting the daemds
+while [ $RETRIES -ne 0 ]; do
+  $DOCKER_EXEC test -S $SOCKET && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means the socket is not present.
+echo "Socket file ${SOCKET} could not be found, which means the Metadata Server is not running. Showing ceph-mds unit logs now:"
+journalctl -u ceph-mds@${MDS_NAME}
+exit 1

roles/ceph-handler/templates/restart_mgr_daemon.sh.j2

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_mgr_check_retries }}"
+DELAY="{{ handler_health_mgr_check_delay }}"
+MGR_NAME="{{ ansible_facts['hostname'] }}"
+{% if containerized_deployment | bool %}
+DOCKER_EXEC="{{ container_binary }} exec ceph-mgr-{{ ansible_facts['hostname'] }}"
+{% endif %}
+
+# Backward compatibility
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['fqdn'] }}.asok
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mgr.{{ ansible_facts['hostname'] }}.asok
+
+systemctl reset-failed ceph-mgr@${MGR_NAME}
+# First, restart the daemon
+systemctl restart ceph-mgr@${MGR_NAME}
+
+# Wait and ensure the socket exists after restarting the daemds
+while [ $RETRIES -ne 0 ]; do
+  $DOCKER_EXEC test -S $SOCKET && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means the socket is not present.
+echo "Socket file ${SOCKET} could not be found, which means ceph manager is not running. Showing ceph-mgr unit logs now:"
+journalctl -u ceph-mgr@${MGR_NAME}
+exit 1

roles/ceph-handler/templates/restart_mon_daemon.sh.j2

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_mon_check_retries }}"
+DELAY="{{ handler_health_mon_check_delay }}"
+MONITOR_NAME="{{ monitor_name }}"
+{% if containerized_deployment | bool %}
+DOCKER_EXEC="{{ container_binary }} exec ceph-mon-{{ ansible_facts['hostname'] }}"
+{% endif %}
+
+# if daemon is uninstalled, no restarting is needed; so exit with success
+systemctl status ceph-mon@{{ ansible_facts['hostname'] }} > /dev/null
+if [[ $? -ne 0 ]]; then
+  exit 0
+fi
+
+# Backward compatibility
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['fqdn'] }}.asok
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-mon.{{ ansible_facts['hostname'] }}.asok
+
+check_quorum() {
+while [ $RETRIES -ne 0 ]; do
+  $DOCKER_EXEC ceph --cluster {{ cluster }} quorum_status --format json | "{{ discovered_interpreter_python }}" -c 'import sys, json; exit(0) if "{{ monitor_name }}" in json.load(sys.stdin)["quorum_names"] else exit(1)' && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means there is a problem with the quorum
+echo "Error with quorum."
+echo "cluster status:"
+$DOCKER_EXEC ceph --cluster {{ cluster }} -s
+echo "quorum status:"
+$DOCKER_EXEC ceph --cluster {{ cluster }} daemon mon.${MONITOR_NAME} mon_status
+$DOCKER_EXEC ceph --cluster {{ cluster }} daemon mon.${MONITOR_NAME} quorum_status
+exit 1
+}
+
+# First, restart the daemon
+systemctl restart ceph-mon@{{ ansible_facts['hostname'] }}
+
+COUNT=10
+# Wait and ensure the socket exists after restarting the daemon
+while [ $COUNT -ne 0 ]; do
+  $DOCKER_EXEC test -S $SOCKET && check_quorum
+  sleep $DELAY
+  let COUNT=COUNT-1
+done
+# If we reach this point, it means the socket is not present.
+echo "Socket file ${SOCKET} could not be found, which means the monitor is not running. Showing ceph-mon unit logs now:"
+journalctl -u ceph-mon@{{ ansible_facts['hostname'] }}
+exit 1

roles/ceph-handler/templates/restart_nfs_daemon.sh.j2

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_nfs_check_retries }}"
+DELAY="{{ handler_health_nfs_check_delay }}"
+NFS_NAME="ceph-nfs@{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}"
+PID=/var/run/ganesha/ganesha.pid
+{% if containerized_deployment | bool %}
+DOCKER_EXEC="{{ container_binary }} exec ceph-nfs-{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}"
+{% endif %}
+
+# First, restart the daemon
+{% if containerized_deployment | bool -%}
+systemctl restart $NFS_NAME
+# Wait and ensure the pid exists after restarting the daemon
+while [ $RETRIES -ne 0 ]; do
+  $DOCKER_EXEC test -f $PID && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means the pid is not present.
+echo "PID file ${PID} could not be found, which means Ganesha is not running. Showing $NFS_NAME unit logs now:"
+journalctl -u $NFS_NAME
+exit 1
+{% else %}
+systemctl restart nfs-ganesha
+{% endif %}

roles/ceph-handler/templates/restart_osd_daemon.sh.j2

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+DELAY="{{ handler_health_osd_check_delay }}"
+CEPH_CLI="--name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring --cluster {{ cluster }}"
+
+check_pgs() {
+  num_pgs=$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')
+  if [[ "$num_pgs" == "0" ]]; then
+    return 0
+  fi
+  while [ $RETRIES -ne 0 ]; do
+    test "$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(json.load(sys.stdin)["pgmap"]["num_pgs"])')" -eq "$($container_exec ceph $CEPH_CLI -s -f json | "{{ discovered_interpreter_python }}" -c 'import sys, json; print(sum ( [ i["count"] for i in json.load(sys.stdin)["pgmap"]["pgs_by_state"] if "active+clean" in i["state_name"]]))')"
+    RET=$?
+    test $RET -eq 0 && return 0
+    sleep $DELAY
+    let RETRIES=RETRIES-1
+  done
+  # PGs not clean, exiting with return code 1
+  echo "Error while running 'ceph $CEPH_CLI -s', PGs were not reported as active+clean"
+  echo "It is possible that the cluster has less OSDs than the replica configuration"
+  echo "Will refuse to continue"
+  $container_exec ceph $CEPH_CLI -s
+  $container_exec ceph $CEPH_CLI osd dump
+  $container_exec ceph $CEPH_CLI osd tree
+  $container_exec ceph $CEPH_CLI osd crush rule dump
+  exit 1
+}
+
+wait_for_socket_in_container() {
+  osd_mount_point=$({{ container_binary }} exec "$1" df --output=target | grep '/var/lib/ceph/osd/')
+  whoami=$({{ container_binary }} exec "$1" cat $osd_mount_point/whoami)
+  if ! {{ container_binary }} exec "$1" timeout 10 bash -c "while [ ! -e /var/run/ceph/{{ cluster }}-osd.${whoami}.asok ]; do sleep 1 ; done"; then
+    echo "Timed out while trying to look for a Ceph OSD socket."
+    echo "Abort mission!"
+    exit 1
+  fi
+}
+
+get_dev_name() {
+  echo $1 | sed -r 's/ceph-osd@([a-z]{1,4})\.service/\1/'
+}
+
+get_container_id_from_dev_name() {
+  local id
+  local count
+  count=10
+  while [ $count -ne 0 ]; do
+    id=$({{ container_binary }} ps | grep -E "${1}$" | cut -d' ' -f1)
+    test "$id" != "" && break
+    sleep $DELAY
+    let count=count-1
+  done
+  echo "$id"
+}
+
+# For containerized deployments, the unit file looks like: ceph-osd@sda.service
+# For non-containerized deployments, the unit file looks like: ceph-osd@NNN.service where NNN is OSD ID
+for unit in $(systemctl list-units | grep -E "loaded * active" | grep -oE "ceph-osd@([0-9]+).service"); do
+  # First, restart daemon(s)
+  systemctl restart "${unit}"
+  # We need to wait because it may take some time for the socket to actually exists
+  COUNT=10
+  # Wait and ensure the socket exists after restarting the daemon
+  {% if containerized_deployment | bool %}
+  osd_id=$(echo ${unit#ceph-osd@} | grep -oE '[0-9]+')
+  container_id=$(get_container_id_from_dev_name "ceph-osd-${osd_id}")
+  container_exec="{{ container_binary }} exec $container_id"
+  {% else %}
+  osd_id=$(echo ${unit#ceph-osd@} | grep -oE '[0-9]+')
+  {% endif %}
+  SOCKET=/var/run/ceph/{{ cluster }}-osd.${osd_id}.asok
+  while [ $COUNT -ne 0 ]; do
+    RETRIES="{{ handler_health_osd_check_retries }}"
+    $container_exec test -S "$SOCKET" && check_pgs && continue 2
+    sleep $DELAY
+    let COUNT=COUNT-1
+  done
+  # If we reach this point, it means the socket is not present.
+  echo "Socket file ${SOCKET} could not be found, which means the osd daemon is not running. Showing ceph-osd unit logs now:"
+  journalctl -u "${unit}"
+  exit 1
+done

roles/ceph-handler/templates/restart_rbd_mirror_daemon.sh.j2

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_rbd_mirror_check_retries }}"
+DELAY="{{ handler_health_rbd_mirror_check_delay }}"
+RBD_MIRROR_NAME="{{ ansible_facts['hostname'] }}"
+{% if containerized_deployment | bool %}
+DOCKER_EXEC="{{ container_binary }} exec ceph-rbd-mirror-{{ ansible_facts['hostname'] }}"
+{% endif %}
+
+# Backward compatibility
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['fqdn'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['fqdn'] }}.asok
+$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['hostname'] }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rbd-mirror.{{ ansible_facts['hostname'] }}.asok
+
+# First, restart the daemon
+systemctl restart ceph-rbd-mirror@rbd-mirror.${RBD_MIRROR_NAME}
+
+# Wait and ensure the socket exists after restarting the daemon
+while [ $RETRIES -ne 0 ]; do
+  $DOCKER_EXEC test -S $SOCKET && exit 0
+  sleep $DELAY
+  let RETRIES=RETRIES-1
+done
+# If we reach this point, it means the socket is not present.
+echo "Socket file ${SOCKET} could not be found, which means rbd mirror is not running. Showing ceph-rbd-mirror unit logs now:"
+journalctl -u ceph-rbd-mirror@rbd-mirror.${RBD_MIRROR_NAME}
+exit 1

roles/ceph-handler/templates/restart_rgw_daemon.sh.j2

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+RETRIES="{{ handler_health_rgw_check_retries }}"
+DELAY="{{ handler_health_rgw_check_delay }}"
+HOST_NAME="{{ ansible_facts['hostname'] }}"
+RGW_NUMS={{ rgw_instances | length | int }}
+RGW_FRONTEND_SSL_CERT={{ radosgw_frontend_ssl_certificate }}
+if [ -n "$RGW_FRONTEND_SSL_CERT" ]; then
+    RGW_PROTOCOL=https
+else
+    RGW_PROTOCOL=http
+fi
+INSTANCES_NAME=({% for i in rgw_instances %}{{ i.instance_name }} {% endfor %})
+HAPROXY_BACKEND=({% for i in rgw_instances %}{{ i.haproxy_backend | default('rgw-backend') }} {% endfor %})
+RGW_IPS=({% for i in rgw_instances %}{{ i.radosgw_address }} {% endfor %})
+RGW_PORTS=({% for i in rgw_instances %}{{ i.radosgw_frontend_port }} {% endfor %})
+RGW_ZONE="{{ rgw_zone }}"
+declare -a DOCKER_EXECS
+declare -a SOCKET_PREFIX
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  SOCKET_PREFIX[i]="/var/run/ceph/{{ cluster }}-client.rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}"
+  DOCKER_EXECS[i]=""
+{% if containerized_deployment | bool %}
+  DOCKER_EXECS[i]="{{ container_binary }} exec ceph-rgw-${RGW_ZONE}-${HOST_NAME}-${INSTANCES_NAME[i]}"
+{% endif %}
+done
+
+check_socket() {
+  local i=$1
+  local succ=0
+  local count=10
+  # Wait and ensure the socket exists after restarting the daemon
+  while [ $count -ne 0 ]; do
+    SOCKET=$(grep ${SOCKET_PREFIX[i]} /proc/net/unix | awk '{ print $8 }')
+    if [ -n "${SOCKET}" ]; then
+      ${DOCKER_EXECS[i]} test -S ${SOCKET} && succ=$((succ+1)) && break
+    fi
+    sleep $DELAY
+    let count=count-1
+  done
+  if [ $succ -ne 1 ]; then
+    echo "Socket file ${SOCKET} could not be found, which means Rados Gateway is not running. Showing ceph-rgw unit logs now:"
+    journalctl -u ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}
+    exit 1
+  fi
+}
+
+check_for_curl_or_wget() {
+  local i=$1
+  url="$RGW_PROTOCOL://${RGW_IPS[i]}:${RGW_PORTS[i]}"
+  if ${DOCKER_EXECS[i]} command -v wget &>/dev/null; then
+    rgw_test_result=$(${DOCKER_EXECS[i]} wget --no-check-certificate --tries 1 --quiet --server-response --spider -O /dev/null 2>&1 $url | awk 'NR==1{print $2}')
+  elif ${DOCKER_EXECS[i]} command -v curl &>/dev/null; then
+    rgw_test_result=$(${DOCKER_EXECS[i]} curl {{ '-g' if ip_version == 'ipv6' else '' }} -k -w "%{http_code}" --silent --output /dev/null $url)
+  else
+    echo "It seems that neither curl nor wget are available on your system."
+    echo "Cannot test rgw connection."
+    rgw_test_result=0
+  fi
+}
+
+check_rest() {
+  local i=$1
+  local succ=0
+  while [ $RETRIES -ne 0 ]; do
+    check_for_curl_or_wget ${i}
+    if [ $rgw_test_result -eq 200 ] || [ $rgw_test_result -eq 404 ] || [ $rgw_test_result -eq 405 ]; then
+      succ=$((succ+1))
+      break
+    fi
+    sleep $DELAY
+    let RETRIES=RETRIES-1
+  done
+  if [ $succ -ne 1 ]; then
+    # If we reach this point, it means there is a problem with the connection to rgw
+    echo "Error connecting locally to Rados Gateway service: $RGW_PROTOCOL://${RGW_IPS[i]}:${RGW_PORTS[i]}"
+    exit 1
+  fi
+}
+
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  # Check if systemd unit exists
+  # This is needed for new instances as the restart might trigger before the deployment
+  if ! systemctl list-units --full --all | grep -q "ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}"; then
+    echo "Systemd unit ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]} does not exist."
+    continue
+  fi
+
+{% if handler_rgw_use_haproxy_maintenance %}
+  # set server weight to 0 on haproxy
+  echo "set weight ${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} 0" | socat stdio {{ haproxy_socket_path }}
+
+  # wait for the connections to drop
+  retries={{ handler_rgw_haproxy_maintenance_retries | default(60) }}
+  while [ $retries -gt 0 ]; do
+    if [ "$(echo "show servers conn ${HAPROXY_BACKEND[i]}" | socat stdio {{ haproxy_socket_path }} | grep "${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} " | awk '{ print $7 }')" -eq 0 ]; then
+      break
+    fi
+    sleep 1
+    let retries=retries-1
+  done
+{% endif %}
+
+  # Restart the daemon
+  systemctl restart ceph-radosgw@rgw.${RGW_ZONE}.${HOST_NAME}.${INSTANCES_NAME[i]}
+
+  # Check socket files
+  check_socket ${i}
+  # Check rest
+  check_rest ${i}
+
+{% if handler_rgw_use_haproxy_maintenance %}
+  # set server weight to 100 on haproxy
+  echo "set weight ${HAPROXY_BACKEND[i]}/${INSTANCES_NAME[i]} 100" | socat stdio {{ haproxy_socket_path }}
+{% endif %}
+done