deep_scrub_choked

This commit is contained in:
a.pivkin
2026-04-06 07:08:17 +03:00
commit 9149dc1d7f
840 changed files with 52471 additions and 0 deletions
@@ -0,0 +1,23 @@
---
- name: Create rgw nfs user "{{ ceph_nfs_rgw_user }}"
radosgw_user:
name: "{{ ceph_nfs_rgw_user }}"
cluster: "{{ cluster }}"
display_name: "RGW NFS User"
access_key: "{{ ceph_nfs_rgw_access_key | default(omit) }}"
secret_key: "{{ ceph_nfs_rgw_secret_key | default(omit) }}"
run_once: true
register: rgw_nfs_user
changed_when: false
delegate_to: "{{ groups[mon_group_name][0] }}"
when: nfs_obj_gw | bool
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
- name: Set_fact ceph_nfs_rgw_access_key and ceph_nfs_rgw_secret_key
ansible.builtin.set_fact:
ceph_nfs_rgw_access_key: "{{ (rgw_nfs_user.stdout | from_json)['keys'][0]['access_key'] }}"
ceph_nfs_rgw_secret_key: "{{ (rgw_nfs_user.stdout | from_json)['keys'][0]['secret_key'] }}"
delegate_to: "{{ groups[mon_group_name][0] }}"
when: nfs_obj_gw | bool
+96
View File
@@ -0,0 +1,96 @@
---
# global/common requirement
- name: Stop nfs server service
ansible.builtin.systemd:
name: "{{ 'nfs-server' if ansible_facts['os_family'] == 'RedHat' else 'nfsserver' if ansible_facts['os_family'] == 'Suse' else 'nfs-kernel-server' if ansible_facts['os_family'] == 'Debian' }}"
state: stopped
enabled: false
failed_when: false
- name: Include pre_requisite_non_container.yml
ansible.builtin.include_tasks: pre_requisite_non_container.yml
when: not containerized_deployment | bool
- name: Include pre_requisite_container.yml
ansible.builtin.include_tasks: pre_requisite_container.yml
when: containerized_deployment | bool
- name: Set_fact _rgw_hostname
ansible.builtin.set_fact:
_rgw_hostname: "{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] }}"
- name: Set rgw parameter (log file)
ceph_config:
action: set
who: "client.rgw.{{ _rgw_hostname }}"
option: "log file"
value: "/var/log/ceph/{{ cluster }}-rgw-{{ hostvars[inventory_hostname]['ansible_facts']['hostname'] }}.log"
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
loop: "{{ groups.get('nfss', []) }}"
- name: Include create_rgw_nfs_user.yml
ansible.builtin.import_tasks: create_rgw_nfs_user.yml
when: groups.get(mon_group_name, []) | length > 0
- name: Install nfs-ganesha-selinux on RHEL 8
ansible.builtin.package:
name: nfs-ganesha-selinux
state: present
register: result
until: result is succeeded
when:
- not containerized_deployment | bool
- inventory_hostname in groups.get(nfs_group_name, [])
- ansible_facts['os_family'] == 'RedHat'
- ansible_facts['distribution_major_version'] == '8'
# NOTE (leseb): workaround for issues with ganesha and librgw
- name: Add ganesha_t to permissive domain
community.general.selinux_permissive:
name: ganesha_t
permissive: true
failed_when: false
when:
- not containerized_deployment | bool
- ansible_facts['os_family'] == 'RedHat'
- ansible_facts['selinux']['status'] == 'enabled'
- name: Nfs with external ceph cluster task related
when:
- groups.get(mon_group_name, []) | length == 0
- ceph_nfs_ceph_user is defined
block:
- name: Create keyring directory
ansible.builtin.file:
path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ item }}"
state: directory
owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
mode: "0755"
with_items:
- "{{ ceph_nfs_ceph_user }}"
- "{{ ansible_facts['hostname'] }}"
- name: Set_fact rgw_client_name
ansible.builtin.set_fact:
rgw_client_name: "client.rgw.{{ ceph_nfs_ceph_user }}"
- name: Get client cephx keys
ansible.builtin.copy:
dest: "{{ item.1 }}"
content: "{{ item.0.content | b64decode }}"
mode: "{{ item.0.item.get('mode', '0600') }}"
owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
with_nested:
- "{{ hostvars[groups['_filtered_clients'][0]]['slurp_client_keys']['results'] | default([]) }}"
- ['/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ceph_nfs_ceph_user }}/keyring', "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}/keyring"]
when:
- not item.0.get('skipped', False)
- item.0.item.name == 'client.' + ceph_nfs_ceph_user or item.0.item.name == rgw_client_name
no_log: "{{ no_log_on_ceph_key_tasks }}"
- name: Include start_nfs.yml
ansible.builtin.import_tasks: start_nfs.yml
@@ -0,0 +1,108 @@
---
- name: Keyring related tasks
when: groups.get(mon_group_name, []) | length > 0
block:
- name: Set_fact container_exec_cmd
ansible.builtin.set_fact:
container_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ hostvars[item]['ansible_facts']['hostname'] }}"
with_items: "{{ groups.get(mon_group_name, []) }}"
delegate_to: "{{ item }}"
delegate_facts: true
run_once: true
- name: Create directories
ansible.builtin.file:
path: "{{ item.0 }}"
state: "directory"
owner: "{{ ceph_uid }}"
group: "{{ ceph_uid }}"
mode: "0755"
delegate_to: "{{ item.1 }}"
with_nested:
- ["/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}",
"/var/lib/ceph/radosgw/{{ cluster }}-nfs.{{ ansible_facts['hostname'] }}"]
- ["{{ groups.get(mon_group_name)[0] }}", "{{ inventory_hostname }}"]
- name: Set_fact keyrings_list
ansible.builtin.set_fact:
keyrings_list:
- { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring", copy_key: "{{ nfs_obj_gw }}" }
- { name: "client.admin", path: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" }
- { name: "client.rgw.{{ ansible_facts['hostname'] }}", create: true, path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}/keyring", caps: { "mon": "allow r", "osd": "allow rwx tag rgw *=*"} }
- { name: "client.nfs.{{ ansible_facts['hostname'] }}", create: true, path: "/var/lib/ceph/radosgw/{{ cluster }}-nfs.{{ ansible_facts['hostname'] }}/keyring", caps: { "mon": "r", "osd": "allow rw pool=.nfs"} }
- name: Create keyrings from a monitor
ceph_key:
name: "{{ item.name }}"
cluster: "{{ cluster }}"
dest: "{{ item.path }}"
caps: "{{ item.caps }}"
import_key: true
owner: "{{ ceph_uid }}"
group: "{{ ceph_uid }}"
mode: "0600"
no_log: "{{ no_log_on_ceph_key_tasks }}"
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment else None }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
delegate_to: "{{ groups.get(mon_group_name)[0] }}"
loop: "{{ keyrings_list }}"
when:
- cephx | bool
- item.create | default(False) | bool
- name: Get keys from monitors
ceph_key_info:
name: "{{ item.name }}"
cluster: "{{ cluster }}"
output_format: plain
state: info
environment:
CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
CEPH_CONTAINER_BINARY: "{{ container_binary }}"
register: _rgw_keys
loop: "{{ keyrings_list }}"
delegate_to: "{{ groups.get(mon_group_name)[0] }}"
run_once: true
when:
- cephx | bool
- item.copy_key | default(True) | bool
no_log: "{{ no_log_on_ceph_key_tasks }}"
- name: Debug
ansible.builtin.debug:
msg: "{{ _rgw_keys }}"
- name: Copy ceph key(s) if needed
ansible.builtin.copy:
dest: "{{ item.item.path }}"
content: "{{ item.stdout + '\n' }}"
owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
mode: "{{ ceph_keyring_permissions }}"
with_items: "{{ _rgw_keys.results }}"
when:
- cephx | bool
- item.item.copy_key | default(True) | bool
no_log: "{{ no_log_on_ceph_key_tasks }}"
- name: Dbus related tasks
when: ceph_nfs_dynamic_exports | bool
block:
- name: Get file
ansible.builtin.command: "{{ container_binary }} run --rm --entrypoint=cat {{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag }} /etc/dbus-1/system.d/org.ganesha.nfsd.conf"
register: dbus_ganesha_file
run_once: true
changed_when: false
- name: Create dbus service file
ansible.builtin.copy:
content: "{{ dbus_ganesha_file.stdout }}"
dest: /etc/dbus-1/system.d/org.ganesha.nfsd.conf
owner: "root"
group: "root"
mode: "0644"
- name: Reload dbus configuration
ansible.builtin.command: "killall -SIGHUP dbus-daemon"
changed_when: false
@@ -0,0 +1,96 @@
---
- name: Include red hat based system related tasks
ansible.builtin.include_tasks: pre_requisite_non_container_red_hat.yml
when: ansible_facts['os_family'] == 'RedHat'
- name: Include debian based system related tasks
ansible.builtin.include_tasks: pre_requisite_non_container_debian.yml
when: ansible_facts['os_family'] == 'Debian'
- name: Install nfs rgw/cephfs gateway - SUSE/openSUSE
community.general.zypper:
name: "{{ item.name }}"
disable_gpg_check: true
with_items:
- { name: 'nfs-ganesha-rgw', install: "{{ nfs_obj_gw }}" }
- { name: 'radosgw', install: "{{ nfs_obj_gw }}" }
- { name: 'nfs-ganesha-ceph', install: "{{ nfs_file_gw }}" }
when:
- (ceph_origin == 'repository' or ceph_origin == 'distro')
- ansible_facts['os_family'] == 'Suse'
- item.install | bool
register: result
until: result is succeeded
# NOTE (leseb): we use root:ceph for permissions since ganesha
# does not have the right selinux context to read ceph directories.
- name: Create rados gateway and ganesha directories
ansible.builtin.file:
path: "{{ item.name }}"
state: directory
owner: "{{ item.owner | default('ceph') }}"
group: "{{ item.group | default('ceph') }}"
mode: "{{ ceph_directories_mode }}"
with_items:
- { name: "/var/lib/ceph/bootstrap-rgw", create: "{{ nfs_obj_gw }}" }
- { name: "/var/lib/ceph/radosgw", create: "{{ nfs_obj_gw }}" }
- { name: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}", create: "{{ nfs_obj_gw }}" }
- { name: "{{ rbd_client_admin_socket_path }}", create: "{{ nfs_obj_gw }}" }
- { name: "/var/log/ceph", create: true }
- { name: "/var/log/ganesha", create: true, owner: root, group: root }
- { name: "/var/run/ceph", create: true }
when: item.create | bool
- name: Cephx related tasks
when:
- cephx | bool
- groups.get(mon_group_name, []) | length > 0
block:
- name: Get keys from monitors
ceph_key_info:
name: "{{ item.name }}"
cluster: "{{ cluster }}"
output_format: plain
state: info
register: _rgw_keys
with_items:
- { name: "client.bootstrap-rgw", path: "/var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring", copy_key: "{{ nfs_obj_gw }}" }
- { name: "client.admin", path: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" }
delegate_to: "{{ groups.get(mon_group_name)[0] }}"
run_once: true
when:
- cephx | bool
- item.copy_key | bool
no_log: "{{ no_log_on_ceph_key_tasks }}"
- name: Copy ceph key(s) if needed
ansible.builtin.copy:
dest: "{{ item.item.path }}"
content: "{{ item.stdout + '\n' }}"
owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
mode: "{{ ceph_keyring_permissions }}"
with_items: "{{ _rgw_keys.results }}"
when:
- cephx | bool
- item.item.copy_key | bool
no_log: "{{ no_log_on_ceph_key_tasks }}"
- name: Nfs object gateway related tasks
when: nfs_obj_gw | bool
block:
- name: Create rados gateway keyring
ceph_key:
name: "client.rgw.{{ ansible_facts['hostname'] }}"
cluster: "{{ cluster }}"
user: client.bootstrap-rgw
user_key: "/var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring"
caps:
mon: "allow rw"
osd: "allow rwx"
dest: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}/keyring"
import_key: false
owner: ceph
group: ceph
mode: "{{ ceph_keyring_permissions }}"
no_log: "{{ no_log_on_ceph_key_tasks }}"
@@ -0,0 +1,80 @@
---
- name: Debian based systems - repo handling
when: ceph_origin == 'repository'
block:
- name: Stable repos specific tasks
when:
- nfs_ganesha_stable | bool
- ceph_repository == 'community'
block:
- name: Add nfs-ganesha stable repository
ansible.builtin.apt_repository:
repo: "deb {{ nfs_ganesha_stable_deb_repo }} {{ ceph_stable_distro_source | default(ansible_facts['distribution_release']) }} main"
state: present
update_cache: false
register: add_ganesha_apt_repo
- name: Add libntirpc stable repository
ansible.builtin.apt_repository:
repo: "deb {{ libntirpc_stable_deb_repo }} {{ ceph_stable_distro_source | default(ansible_facts['distribution_release']) }} main"
state: present
update_cache: false
register: add_libntirpc_apt_repo
when: libntirpc_stable_deb_repo is defined
- name: Add nfs-ganesha ppa apt key
ansible.builtin.apt_key:
keyserver: "{{ nfs_ganesha_apt_keyserver }}"
id: "{{ nfs_ganesha_apt_key_id }}"
when:
- nfs_ganesha_apt_key_id is defined
- nfs_ganesha_apt_keyserver is defined
- name: Update apt cache
ansible.builtin.apt:
update_cache: true
register: update_ganesha_apt_cache
retries: 5
delay: 2
until: update_ganesha_apt_cache is success
when: add_ganesha_apt_repo is changed or add_libntirpc_apt_repo is changed
- name: Debian based systems - dev repos specific tasks
when:
- nfs_ganesha_dev | bool
- ceph_repository == 'dev'
block:
- name: Fetch nfs-ganesha development repository
ansible.builtin.uri:
url: "https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/{{ ansible_facts['distribution'] | lower }}/{{ ansible_facts['distribution_release'] }}/flavors/{{ nfs_ganesha_flavor }}/repo?arch={{ ansible_facts['architecture'] }}"
return_content: true
register: nfs_ganesha_dev_apt_repo
- name: Add nfs-ganesha development repository
ansible.builtin.copy:
content: "{{ nfs_ganesha_dev_apt_repo.content }}"
dest: /etc/apt/sources.list.d/nfs-ganesha-dev.list
owner: root
group: root
backup: true
mode: "0644"
- name: Debain based systems - install required packages
block:
- name: Debian based systems
when: ceph_origin == 'repository' or ceph_origin == 'distro'
block:
- name: Install nfs rgw/cephfs gateway - debian
ansible.builtin.apt:
name: ['nfs-ganesha-rgw', 'radosgw']
allow_unauthenticated: true
register: result
until: result is succeeded
when: nfs_obj_gw | bool
- name: Install nfs rgw/cephfs gateway - debian
ansible.builtin.apt:
name: nfs-ganesha-ceph
allow_unauthenticated: true
register: result
until: result is succeeded
when: nfs_file_gw | bool
@@ -0,0 +1,43 @@
---
- name: Red hat based systems - repo handling
when: ceph_origin == 'repository'
block:
- name: Red hat based systems - stable repo related tasks
when:
- nfs_ganesha_stable | bool
- ceph_repository == 'community'
block:
- name: Add nfs-ganesha stable repository
ansible.builtin.package:
name: "{{ centos_release_nfs }}"
state: present
- name: Red hat based systems - dev repo related tasks
when:
- nfs_ganesha_dev | bool
- ceph_repository == 'dev'
block:
- name: Add nfs-ganesha dev repo
ansible.builtin.get_url:
url: "https://shaman.ceph.com/api/repos/nfs-ganesha/next/latest/{{ ansible_facts['distribution'] | lower }}/{{ ansible_facts['distribution_major_version'] }}/flavors/{{ nfs_ganesha_flavor }}/repo?arch={{ ansible_facts['architecture'] }}"
dest: /etc/yum.repos.d/nfs-ganesha-dev.repo
mode: "0644"
force: true
- name: Red hat based systems - install nfs packages
block:
- name: Install nfs cephfs gateway
ansible.builtin.package:
name: ['nfs-ganesha-ceph', 'nfs-ganesha-rados-grace']
state: "{{ (upgrade_ceph_packages | bool) | ternary('latest', 'present') }}"
register: result
until: result is succeeded
when: nfs_file_gw | bool
- name: Install redhat nfs-ganesha-rgw and ceph-radosgw packages
ansible.builtin.package:
name: ['nfs-ganesha-rgw', 'nfs-ganesha-rados-grace', 'nfs-ganesha-rados-urls', 'ceph-radosgw']
state: "{{ (upgrade_ceph_packages | bool) | ternary('latest', 'present') }}"
register: result
until: result is succeeded
when: nfs_obj_gw | bool
+105
View File
@@ -0,0 +1,105 @@
---
- name: Nfs various pre-requisites tasks
block:
- name: Set_fact exec_cmd_nfs - external
ansible.builtin.set_fact:
exec_cmd_nfs: "{{ container_binary + ' run --rm --net=host -v /etc/ceph:/etc/ceph:z -v /var/lib/ceph/:/var/lib/ceph/:z -v /var/log/ceph/:/var/log/ceph/:z --entrypoint=rados ' + ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else 'rados' }} -n client.{{ ceph_nfs_ceph_user }} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ceph_nfs_ceph_user }}/keyring"
delegate_node: "{{ inventory_hostname }}"
when: groups.get(mon_group_name, []) | length == 0
- name: Set_fact exec_cmd_nfs - internal
ansible.builtin.set_fact:
exec_cmd_nfs: "{{ container_binary + ' exec ceph-mon-' + hostvars[groups[mon_group_name][0]]['ansible_facts']['hostname'] if containerized_deployment | bool else '' }} rados"
delegate_node: "{{ groups[mon_group_name][0] }}"
when: groups.get(mon_group_name, []) | length > 0
- name: Check if rados index object exists
ansible.builtin.shell: "set -o pipefail && {{ exec_cmd_nfs | default('') }} -p {{ cephfs_data_pool.name }} --cluster {{ cluster }} ls | grep {{ ceph_nfs_rados_export_index }}"
changed_when: false
failed_when: false
register: rados_index_exists
check_mode: false
when: ceph_nfs_rados_backend | bool
delegate_to: "{{ delegate_node }}"
run_once: true
- name: Create an empty rados index object
ansible.builtin.command: "{{ exec_cmd_nfs | default('') }} -p {{ cephfs_data_pool.name }} --cluster {{ cluster }} put {{ ceph_nfs_rados_export_index }} /dev/null"
when:
- ceph_nfs_rados_backend | bool
- rados_index_exists.rc != 0
delegate_to: "{{ delegate_node }}"
changed_when: false
run_once: true
- name: Create /etc/ganesha
ansible.builtin.file:
path: /etc/ganesha
state: directory
owner: root
group: root
mode: "0755"
- name: Generate ganesha configuration file
ansible.builtin.template:
src: "ganesha.conf.j2"
dest: /etc/ganesha/ganesha.conf
owner: "root"
group: "root"
mode: "0644"
notify: Restart ceph nfss
- name: Generate ganesha idmap.conf file
openstack.config_template.config_template:
src: "idmap.conf.j2"
dest: "{{ ceph_nfs_idmap_conf }}"
owner: "root"
group: "root"
mode: "0644"
config_overrides: "{{ idmap_conf_overrides }}"
config_type: ini
notify: Restart ceph nfss
- name: Create exports directory
ansible.builtin.file:
path: /etc/ganesha/export.d
state: directory
owner: "root"
group: "root"
mode: "0755"
when: ceph_nfs_dynamic_exports | bool
- name: Create exports dir index file
ansible.builtin.copy:
content: ""
force: false
dest: /etc/ganesha/export.d/INDEX.conf
owner: "root"
group: "root"
mode: "0644"
when: ceph_nfs_dynamic_exports | bool
- name: Include_tasks systemd.yml
ansible.builtin.include_tasks: systemd.yml
when: containerized_deployment | bool
- name: Systemd start nfs container
ansible.builtin.systemd:
name: ceph-nfs@{{ ceph_nfs_service_suffix | default(ansible_facts['hostname']) }}
state: started
enabled: true
masked: false
daemon_reload: true
when:
- containerized_deployment | bool
- ceph_nfs_enable_service | bool
- name: Start nfs gateway service
ansible.builtin.systemd:
name: nfs-ganesha
state: started
enabled: true
masked: false
when:
- not containerized_deployment | bool
- ceph_nfs_enable_service | bool
+9
View File
@@ -0,0 +1,9 @@
---
- name: Generate systemd unit file
ansible.builtin.template:
src: "{{ role_path }}/templates/ceph-nfs.service.j2"
dest: /etc/systemd/system/ceph-nfs@.service
owner: "root"
group: "root"
mode: "0644"
notify: Restart ceph nfss