deep_scrub_choked

roles/ceph-rgw-loadbalancer/defaults/main.yml

@@ -0,0 +1,26 @@
+---
+# You can override vars by using host or group vars
+
+###########
+# GENERAL #
+###########
+
+haproxy_frontend_port: 80
+haproxy_frontend_ssl_port: 443
+haproxy_frontend_ssl_certificate:
+haproxy_ssl_dh_param: 4096
+haproxy_ssl_ciphers:
+  - EECDH+AESGCM
+  - EDH+AESGCM
+haproxy_ssl_options:
+  - no-sslv3
+  - no-tlsv10
+  - no-tlsv11
+  - no-tls-tickets
+#
+# virtual_ips:
+#    - 192.168.238.250
+#    - 192.168.238.251
+#
+# virtual_ip_netmask: 24
+# virtual_ip_interface: ens33

roles/ceph-rgw-loadbalancer/handlers/main.yml

@@ -0,0 +1,10 @@
+---
+- name: Restart haproxy
+  ansible.builtin.service:
+    name: haproxy
+    state: restarted
+
+- name: Restart keepalived
+  ansible.builtin.service:
+    name: keepalived
+    state: restarted

roles/ceph-rgw-loadbalancer/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Gui Hecheng
+  description: Config HAProxy & Keepalived
+  license: Apache
+  min_ansible_version: '2.8'
+  platforms:
+    - name: EL
+      versions:
+        - 'all'
+  galaxy_tags:
+    - system
+dependencies: []

roles/ceph-rgw-loadbalancer/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Include_tasks pre_requisite.yml
+  ansible.builtin.include_tasks: pre_requisite.yml
+
+- name: Include_tasks start_rgw_loadbalancer.yml
+  ansible.builtin.include_tasks: start_rgw_loadbalancer.yml

roles/ceph-rgw-loadbalancer/tasks/pre_requisite.yml

@@ -0,0 +1,50 @@
+---
+- name: Install haproxy and keepalived
+  ansible.builtin.package:
+    name: ['haproxy', 'keepalived']
+    state: present
+  register: result
+  until: result is succeeded
+
+- name: Generate haproxy configuration file haproxy.cfg
+  ansible.builtin.template:
+    src: haproxy.cfg.j2
+    dest: /etc/haproxy/haproxy.cfg
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    validate: "haproxy -f %s -c"
+  notify: Restart haproxy
+
+- name: Set_fact vip to vrrp_instance
+  ansible.builtin.set_fact:
+    vrrp_instances: "{{ vrrp_instances | default([]) | union([{'name': 'VI_' + index | string, 'vip': item, 'master': groups[rgwloadbalancer_group_name][index]}]) }}"
+  loop: "{{ virtual_ips | flatten(levels=1) }}"
+  loop_control:
+    index_var: index
+
+- name: Generate keepalived configuration file keepalived.conf
+  ansible.builtin.template:
+    src: keepalived.conf.j2
+    dest: /etc/keepalived/keepalived.conf
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  notify: Restart keepalived
+
+- name: Selinux related tasks
+  when:
+    - ansible_facts['os_family'] == 'RedHat'
+    - ansible_facts['selinux']['status'] == 'enabled'
+  block:
+    - name: Set_fact rgw_ports
+      ansible.builtin.set_fact:
+        rgw_ports: "{{ rgw_ports | default([]) | union(hostvars[item]['rgw_instances'] | map(attribute='radosgw_frontend_port') | map('string') | list) }}"
+      with_items: "{{ groups.get(rgw_group_name, []) }}"
+
+    - name: Add selinux rules
+      community.general.seport:
+        ports: "{{ rgw_ports }}"
+        proto: tcp
+        setype: http_port_t
+        state: present

roles/ceph-rgw-loadbalancer/tasks/start_rgw_loadbalancer.yml

@@ -0,0 +1,12 @@
+---
+- name: Start haproxy
+  ansible.builtin.service:
+    name: haproxy
+    state: started
+    enabled: true
+
+- name: Start keepalived
+  ansible.builtin.service:
+    name: keepalived
+    state: started
+    enabled: true

roles/ceph-rgw-loadbalancer/templates/haproxy.cfg.j2

@@ -0,0 +1,63 @@
+# {{ ansible_managed  }}
+global
+    log         127.0.0.1 local2
+
+    chroot      /var/lib/haproxy
+    pidfile     /var/run/haproxy.pid
+    maxconn     8000
+    user        haproxy
+    group       haproxy
+    daemon
+    stats socket /var/lib/haproxy/stats
+{% if haproxy_frontend_ssl_certificate %}
+    tune.ssl.default-dh-param {{ haproxy_ssl_dh_param }}
+    ssl-default-bind-ciphers {{ haproxy_ssl_ciphers | join(':') }}
+    ssl-default-bind-options {{ haproxy_ssl_options | join(' ') }}
+{% endif %}
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 3
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           10s
+    maxconn                 8000
+
+frontend rgw-frontend
+{% if haproxy_frontend_ssl_certificate %}
+    bind *:{{ haproxy_frontend_ssl_port }} ssl crt {{ haproxy_frontend_ssl_certificate }}
+{% else %}
+    bind *:{{ haproxy_frontend_port }}
+{% endif %}
+    default_backend rgw-backend
+
+# when running in an selinux environment, selinux restricts the ports that haproxy can
+# connect to to:
+#  * 80, 81, 443, 488, 8008, 8009, 8443, 9000 (http_port_t) and,
+#  * 8080, 8118, 8123, 10001-10010 (http_cache_port_t)
+#
+# Practically speaking, it would be preferable (and perhaps easier) to configure the
+# rgw daemons to listen on ports 10001-10010 and configure haproxy here to match.
+#
+# Alternatively you can add other unused ports to http_port_t or http_cache_port_t
+# with, e.g.: `semanage port -a -t http_cache_port_t -p tcp 8085`
+# (Note that ports 8081-8084 are already taken and can't be used for haproxy.)
+#
+backend rgw-backend
+    option forwardfor
+    balance static-rr
+    option httpchk HEAD /
+{% for host in groups[rgw_group_name] %}
+{% for instance in hostvars[host]['rgw_instances'] %}
+	server {{ 'server-' + hostvars[host]['ansible_facts']['hostname'] + '-' + instance['instance_name'] }} {{ instance['radosgw_address'] }}:{{ instance['radosgw_frontend_port'] }} weight 100 check
+{% endfor %}
+{% endfor %}

roles/ceph-rgw-loadbalancer/templates/keepalived.conf.j2

@@ -0,0 +1,35 @@
+# {{ ansible_managed }}
+! Configuration File for keepalived
+
+global_defs {
+   router_id CEPH_RGW
+}
+
+vrrp_script check_haproxy {
+    script "killall -0 haproxy"
+    weight -20
+    interval 2
+    rise 2
+    fall 2
+}
+
+{% for instance in vrrp_instances %}
+vrrp_instance {{ instance['name'] }} {
+    state {{ 'MASTER' if inventory_hostname == instance['master'] else 'BACKUP' }}
+    priority {{ '100' if inventory_hostname == instance['master'] else '90' }}
+    interface {{ virtual_ip_interface }}
+    virtual_router_id {{ 50 + loop.index }}
+    advert_int 1
+    authentication {
+        auth_type PASS
+        auth_pass 1234
+    }
+    virtual_ipaddress {
+        {{ instance['vip'] }}/{{ virtual_ip_netmask }} dev {{ virtual_ip_interface }}
+    }
+    track_script {
+        check_haproxy
+    }
+}
+
+{% endfor %}