deep_scrub_choked

2026-04-06 07:08:17 +03:00
commit 9149dc1d7f
840 changed files with 52471 additions and 0 deletions
@@ -0,0 +1,46 @@
+---
+- name: Create bootstrap-osd and osd directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    mode: "{{ ceph_directories_mode }}"
+  when: cephx | bool
+  with_items:
+    - /var/lib/ceph/bootstrap-osd/
+    - /var/lib/ceph/osd/
+
+- name: Get keys from monitors
+  ceph_key_info:
+    name: "{{ item.name }}"
+    cluster: "{{ cluster }}"
+    output_format: plain
+    state: info
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  register: _osd_keys
+  with_items:
+    - { name: "client.bootstrap-osd", path: "/var/lib/ceph/bootstrap-osd/{{ cluster }}.keyring", copy_key: true }
+    - { name: "client.admin", path: "/etc/ceph/{{ cluster }}.client.admin.keyring", copy_key: "{{ copy_admin_key }}" }
+  delegate_to: "{{ groups.get(mon_group_name)[0] }}"
+  run_once: true
+  no_log: "{{ no_log_on_ceph_key_tasks }}"
+  when:
+    - cephx | bool
+    - item.copy_key | bool
+
+- name: Copy ceph key(s) if needed
+  ansible.builtin.copy:
+    dest: "{{ item.item.path }}"
+    content: "{{ item.stdout + '\n' }}"
+    owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    mode: "{{ ceph_keyring_permissions }}"
+  with_items: "{{ _osd_keys.results }}"
+  when:
+    - cephx | bool
+    - item is not skipped
+    - item.item.copy_key | bool
+  no_log: "{{ no_log_on_ceph_key_tasks }}"
@@ -0,0 +1,98 @@
+---
+- name: Configure crush hierarchy
+  ceph_crush:
+    cluster: "{{ cluster }}"
+    location: "{{ osd_crush_location }}"
+    containerized: "{{ hostvars[groups[mon_group_name][0]]['container_exec_cmd'] | default('') }}"
+  register: config_crush_hierarchy
+  delegate_to: '{{ groups[mon_group_name][0] }}'
+  when:
+    - hostvars[groups[mon_group_name][0]]['create_crush_tree'] | default(create_crush_tree) | bool
+    - osd_crush_location is defined
+
+- name: Create configured ec profiles
+  ceph_ec_profile:
+    name: "{{ item.key }}"
+    cluster: "{{ cluster }}"
+    state: "{{ item.value.state | default('present') }}"
+    stripe_unit: "{{ item.value.stripe_unit | default(omit) }}"
+    plugin: "{{ item.value.plugin | default(omit) }}"
+    k: "{{ item.value.k }}"
+    m: "{{ item.value.m }}"
+    d: "{{ item.value.d | default(omit) }}"
+    l: "{{ item.value.l | default(omit) }}"
+    c: "{{ item.value.c | default(omit) }}"
+    scalar_mds: "{{ item.value.scalar_mds | default(omit) }}"
+    technique: "{{ item.value.technique | default(omit) }}"
+    crush_root: "{{ item.value.crush_root | default(omit) }}"
+    crush_failure_domain: "{{ item.value.crush_failure_domain | default(omit) }}"
+    crush_device_class: "{{ item.value.crush_device_class | default(omit) }}"
+    force: "{{ item.value.force | default(false) }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  loop: "{{ ceph_ec_profiles | dict2items }}"
+  delegate_to: '{{ groups[mon_group_name][0] }}'
+  run_once: true
+
+- name: Create configured crush rules
+  ceph_crush_rule:
+    name: "{{ item.name }}"
+    cluster: "{{ cluster }}"
+    rule_type: "{{ item.rule_type | default('replicated') }}"
+    profile: "{{ item.ec_profile | default(omit) }}"
+    bucket_root: "{{ item.root | default(omit) }}"
+    bucket_type: "{{ item.type | default(omit) }}"
+    device_class: "{{ item.class | default(omit) }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  with_items: "{{ hostvars[groups[mon_group_name][0]]['crush_rules'] | default(crush_rules) | unique }}"
+  delegate_to: '{{ groups[mon_group_name][0] }}'
+  run_once: true
+
+- name: Get id for new default crush rule
+  ceph_crush_rule_info:
+    name: "{{ item.name }}"
+    cluster: "{{ cluster }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  register: info_ceph_default_crush_rule
+  with_items: "{{ hostvars[groups[mon_group_name][0]]['crush_rules'] | default(crush_rules) | unique }}"
+  delegate_to: '{{ groups[mon_group_name][0] }}'
+  run_once: true
+  when: item.default | default(False) | bool
+
+# If multiple rules are set as default (should not be) then the last one is taken as actual default.
+# the with_items statement overrides each iteration with the new one.
+# NOTE(leseb): we should actually fail if multiple rules are set as default
+- name: Set_fact info_ceph_default_crush_rule_yaml, ceph_osd_pool_default_crush_rule_name
+  ansible.builtin.set_fact:
+    info_ceph_default_crush_rule_yaml: "{{ item.stdout | default('{}', True) | from_json() }}"
+    ceph_osd_pool_default_crush_rule_name: "{{ (item.stdout | default('{}', True) | from_json).get('rule_name') }}"
+  with_items: "{{ info_ceph_default_crush_rule.results }}"
+  run_once: true
+  when: not item.get('skipped', false)
+
+- name: Insert new default crush rule into daemon to prevent restart
+  ansible.builtin.command: "{{ hostvars[item]['container_exec_cmd'] | default('') }} ceph --admin-daemon /var/run/ceph/{{ cluster }}-mon.{{ hostvars[item]['monitor_name'] }}.asok config set osd_pool_default_crush_rule {{ info_ceph_default_crush_rule_yaml.rule_id }}"
+  changed_when: false
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups[mon_group_name] }}"
+  run_once: true
+  when:
+    - info_ceph_default_crush_rule_yaml | default('') | length > 0
+
+- name: Add new default crush rule to ceph config file
+  community.general.ini_file:
+    dest: "/etc/ceph/{{ cluster }}.conf"
+    section: "global"
+    option: "osd_pool_default_crush_rule"
+    value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}"
+    mode: "0644"
+  delegate_to: "{{ item }}"
+  with_items: "{{ groups[mon_group_name] }}"
+  run_once: true
+  when:
+    - info_ceph_default_crush_rule_yaml | default('') | length > 0
@@ -0,0 +1,101 @@
+---
+- name: Set_fact add_osd
+  ansible.builtin.set_fact:
+    add_osd: "{{ groups[osd_group_name] | length != ansible_play_hosts_all | length }}"
+
+- name: Set_fact container_exec_cmd
+  ansible.builtin.set_fact:
+    container_exec_cmd: "{{ container_binary }} exec ceph-mon-{{ hostvars[item]['ansible_facts']['hostname'] }}"
+  with_items: "{{ groups.get(mon_group_name, []) }}"
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  run_once: true
+  when: containerized_deployment | bool
+
+- name: Include_tasks system_tuning.yml
+  ansible.builtin.include_tasks: system_tuning.yml
+
+- name: Install dependencies
+  ansible.builtin.package:
+    name: parted
+    state: present
+  register: result
+  until: result is succeeded
+  when:
+    - not containerized_deployment | bool
+    - ansible_facts['os_family'] != 'ClearLinux'
+
+- name: Install numactl when needed
+  ansible.builtin.package:
+    name: numactl
+  register: result
+  until: result is succeeded
+  when:
+    - containerized_deployment | bool
+    - ceph_osd_numactl_opts | length > 0
+  tags: with_pkg
+
+- name: Include_tasks common.yml
+  ansible.builtin.include_tasks: common.yml
+
+- name: Set noup flag
+  ceph_osd_flag:
+    name: noup
+    cluster: "{{ cluster }}"
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  run_once: true
+  when:
+    - not rolling_update | default(False) | bool
+    - not switch_to_containers | default(False) | bool
+
+- name: Include_tasks scenarios/lvm.yml
+  ansible.builtin.include_tasks: scenarios/lvm.yml
+  when:
+    - lvm_volumes|length > 0
+    - not rolling_update|default(False) | bool
+
+- name: Include_tasks scenarios/lvm-batch.yml
+  ansible.builtin.include_tasks: scenarios/lvm-batch.yml
+  when:
+    - devices|length > 0
+    - not rolling_update|default(False) | bool
+
+- name: Include_tasks start_osds.yml
+  ansible.builtin.include_tasks: start_osds.yml
+
+- name: Unset noup flag
+  ceph_osd_flag:
+    name: noup
+    cluster: "{{ cluster }}"
+    state: absent
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  when:
+    - not rolling_update | default(False) | bool
+    - not switch_to_containers | default(False) | bool
+    - inventory_hostname == ansible_play_hosts_all | last
+
+- name: Wait for all osd to be up
+  ansible.builtin.command: "{{ hostvars[groups[mon_group_name][0]]['container_exec_cmd'] | default('') }} ceph --cluster {{ cluster }} osd stat -f json"
+  register: wait_for_all_osds_up
+  retries: "{{ nb_retry_wait_osd_up }}"
+  delay: "{{ delay_wait_osd_up }}"
+  changed_when: false
+  delegate_to: "{{ groups[mon_group_name][0] }}"
+  until:
+    - (wait_for_all_osds_up.stdout | from_json)["num_osds"] | int > 0
+    - (wait_for_all_osds_up.stdout | from_json)["num_osds"] == (wait_for_all_osds_up.stdout | from_json)["num_up_osds"]
+  when:
+    - not ansible_check_mode
+    - inventory_hostname == ansible_play_hosts_all | last
+  tags: wait_all_osds_up
+
+- name: Include crush_rules.yml
+  ansible.builtin.include_tasks: crush_rules.yml
+  when: hostvars[groups[mon_group_name][0]]['crush_rule_config'] | default(crush_rule_config) | bool
+  tags: wait_all_osds_up
@@ -0,0 +1,21 @@
+---
+
+- name: Use ceph-volume lvm batch to create osds
+  ceph_volume:
+    cluster: "{{ cluster }}"
+    objectstore: "{{ osd_objectstore }}"
+    batch_devices: "{{ _devices }}"
+    dmcrypt: "{{ dmcrypt | default(omit) }}"
+    crush_device_class: "{{ crush_device_class | default(omit) }}"
+    osds_per_device: "{{ osds_per_device }}"
+    block_db_size: "{{ block_db_size }}"
+    block_db_devices: "{{ dedicated_devices | unique if dedicated_devices | length > 0 else omit }}"
+    wal_devices: "{{ bluestore_wal_devices | unique if bluestore_wal_devices | length > 0 else omit }}"
+    action: "batch"
+  environment:
+    CEPH_VOLUME_DEBUG: "{{ ceph_volume_debug }}"
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+    PYTHONIOENCODING: utf-8
+  when: _devices | default([]) | length > 0
+  tags: prepare_osd
@@ -0,0 +1,21 @@
+---
+- name: Use ceph-volume to create osds
+  ceph_volume:
+    cluster: "{{ cluster }}"
+    objectstore: "{{ osd_objectstore }}"
+    data: "{{ item.data }}"
+    data_vg: "{{ item.data_vg | default(omit) }}"
+    db: "{{ item.db | default(omit) }}"
+    db_vg: "{{ item.db_vg | default(omit) }}"
+    wal: "{{ item.wal | default(omit) }}"
+    wal_vg: "{{ item.wal_vg | default(omit) }}"
+    crush_device_class: "{{ item.crush_device_class | default(crush_device_class) | default(omit) }}"
+    dmcrypt: "{{ dmcrypt | default(omit) }}"
+    action: "{{ 'prepare' if containerized_deployment | bool else 'create' }}"
+  environment:
+    CEPH_VOLUME_DEBUG: "{{ ceph_volume_debug }}"
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+    PYTHONIOENCODING: utf-8
+  with_items: "{{ lvm_volumes }}"
+  tags: prepare_osd
@@ -0,0 +1,69 @@
+---
+# this is for ceph-disk, the ceph-disk command is gone so we have to list /var/lib/ceph
+- name: Get osd ids
+  ansible.builtin.shell: ls /var/lib/ceph/osd/ | sed 's/.*-//'  # noqa risky-shell-pipe
+  args:
+    executable: /bin/bash
+  changed_when: false
+  failed_when: false
+  register: osd_ids_non_container
+
+- name: Collect osd ids
+  ceph_volume:
+    cluster: "{{ cluster }}"
+    action: list
+  environment:
+    CEPH_CONTAINER_IMAGE: "{{ ceph_docker_registry + '/' + ceph_docker_image + ':' + ceph_docker_image_tag if containerized_deployment | bool else None }}"
+    CEPH_CONTAINER_BINARY: "{{ container_binary }}"
+  register: ceph_osd_ids
+
+- name: Include_tasks systemd.yml
+  ansible.builtin.include_tasks: systemd.yml
+  when: containerized_deployment | bool
+
+- name: Ensure systemd service override directory exists
+  ansible.builtin.file:
+    state: directory
+    path: "/etc/systemd/system/ceph-osd@.service.d/"
+    mode: "0755"
+  when:
+    - ceph_osd_systemd_overrides is defined
+    - ansible_facts['service_mgr'] == 'systemd'
+
+- name: Add ceph-osd systemd service overrides
+  openstack.config_template.config_template:
+    src: "ceph-osd.service.d-overrides.j2"
+    dest: "/etc/systemd/system/ceph-osd@.service.d/ceph-osd-systemd-overrides.conf"
+    config_overrides: "{{ ceph_osd_systemd_overrides | default({}) }}"
+    config_type: "ini"
+  when:
+    - ceph_osd_systemd_overrides is defined
+    - ansible_facts['service_mgr'] == 'systemd'
+
+- name: Ensure /var/lib/ceph/osd/<cluster>-<item> is present
+  ansible.builtin.file:
+    state: directory
+    path: "/var/lib/ceph/osd/{{ cluster }}-{{ item }}"
+    mode: "{{ ceph_directories_mode }}"
+    owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+  with_items: "{{ ((ceph_osd_ids.stdout | default('{}', True) | from_json).keys() | list) | union(osd_ids_non_container.stdout_lines | default([])) }}"
+
+- name: Write run file in /var/lib/ceph/osd/xxxx/run
+  ansible.builtin.template:
+    src: systemd-run.j2
+    dest: "/var/lib/ceph/osd/{{ cluster }}-{{ item }}/run"
+    mode: "0700"
+    owner: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment | bool else 'ceph' }}"
+  with_items: "{{ ((ceph_osd_ids.stdout | default('{}', True) | from_json).keys() | list) | union(osd_ids_non_container.stdout_lines | default([])) }}"
+  when: containerized_deployment | bool
+
+- name: Systemd start osd
+  ansible.builtin.systemd:
+    name: ceph-osd@{{ item }}
+    state: started
+    enabled: true
+    masked: false
+    daemon_reload: true
+  with_items: "{{ ((ceph_osd_ids.stdout | default('{}', True) | from_json).keys() | list) | union(osd_ids_non_container.stdout_lines | default([])) }}"
@@ -0,0 +1,43 @@
+---
+- name: Create tmpfiles.d directory
+  ansible.builtin.file:
+    path: "/etc/tmpfiles.d"
+    state: "directory"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  register: "tmpfiles_d"
+  when: disable_transparent_hugepage | bool
+
+- name: Disable transparent hugepage
+  ansible.builtin.template:
+    src: "tmpfiles_hugepage.j2"
+    dest: "/etc/tmpfiles.d/ceph_transparent_hugepage.conf"
+    group: "root"
+    owner: "root"
+    mode: "0644"
+    force: "yes"
+    validate: "systemd-tmpfiles --create %s"
+  when: disable_transparent_hugepage | bool
+
+- name: Get default vm.min_free_kbytes
+  ansible.builtin.slurp:
+    src: /proc/sys/vm/min_free_kbytes
+  register: default_vm_min_free_kbytes
+
+- name: Set_fact vm_min_free_kbytes
+  ansible.builtin.set_fact:
+    vm_min_free_kbytes: "{{ 4194303 if ansible_facts['memtotal_mb'] >= 49152 else default_vm_min_free_kbytes.content | b64decode | trim }}"
+
+- name: Apply operating system tuning
+  ansible.posix.sysctl:
+    name: "{{ item.name }}"
+    value: "{{ item.value }}"
+    state: present
+    sysctl_file: /etc/sysctl.d/ceph-tuning.conf
+    sysctl_set: true
+    ignoreerrors: true
+  with_items:
+    - { name: "fs.aio-max-nr", value: "1048576", enable: "{{ osd_objectstore == 'bluestore' }}" }
+    - "{{ os_tuning_params }}"
+  when: item.enable | default(true) | bool
@@ -0,0 +1,23 @@
+---
+- name: Generate systemd unit file
+  ansible.builtin.template:
+    src: "{{ role_path }}/templates/ceph-osd.service.j2"
+    dest: /etc/systemd/system/ceph-osd@.service
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  notify: Restart ceph osds
+
+- name: Generate systemd ceph-osd target file
+  ansible.builtin.copy:
+    src: ceph-osd.target
+    dest: /etc/systemd/system/ceph-osd.target
+    mode: "0644"
+  when: containerized_deployment | bool
+
+- name: Enable ceph-osd.target
+  ansible.builtin.service:
+    name: ceph-osd.target
+    enabled: true
+    daemon_reload: true
+  when: containerized_deployment | bool