improvements on air gapped installation

basics.yaml

@@ -0,0 +1,67 @@
+    - name: download asc and cephadm for ceph
+      get_url:
+        url: "{{ item.url }}"
+        dest: "{{ item.dest }}"
+      loop:
+        - { url: "https://nexus.test.repo.int.nt-com.ru/repository/keys/ceph/release.asc", dest: "/tmp" }
+
+    - name: Delete previously install ceph.gpg
+      file:
+        path: /usr/share/keyrings/ceph.gpg
+        state: absent
+
+    - name: import downloaded asc
+      shell:
+        chdir: /tmp
+        cmd: echo y | gpg -o /usr/share/keyrings/ceph.gpg --dearmor release.asc;
+
+    - name: Add ceph repo
+      apt_repository:
+        filename: ceph
+        repo: "deb [signed-by=/usr/share/keyrings/ceph.gpg] {{ ceph_repo }} {{ ansible_distribution_release }} main"
+        state: present
+
+    - name: install packages
+      apt:
+        pkg:
+          - ceph-common
+          - cephadm
+          - lvm2
+          - podman
+          - chrony
+          - mc
+        state: present
+
+    - name: Setup ntp client
+      template:
+        src: chrony.yaml.j2
+        dest: /etc/chrony/chrony.conf
+      notify:
+        - ceph-rollout:restart_chrony
+
+    - name: Templating cluster settings files
+      template:
+        src:  "{{ item.name }}"
+        dest: "{{ item.dest }}"
+        trim_blocks: true
+        lstrip_blocks: true
+      loop:
+        - { name: "ceph.conf.j2", dest: "/root/ceph.conf" }
+        - { name: "cluster_spec.yml.j2", dest: "/root/cluster_spec.yml" }
+        - { name: "initial-ceph.j2", dest: "/root/initial-ceph.conf" }
+        - { name: "sshd_config.j2", dest: "/etc/ssh/sshd_config"}
+      run_once: true
+      delegate_to: "{{ ansible_play_hosts[0] }}"
+      notify:
+        - ceph-rollout:restart_sshd
+
+    # UNCOMMENT ONLY IN CASE OF NO ACCESS TO REPOS AND CANNOT DOWNLOAD CEPHADM
+    #    - name: copy onto target host
+    #      copy:
+    #        src: "cephadm"
+    #        dest: "/root/cephadm"
+    #        mode: '0777'
+    #      delegate_to: "{{ ansible_play_hosts[0] }}"
+
+    - name: Flush restart handlers
+      meta: flush_handlers

host_vars/ceph-node01.yaml

@@ -1,3 +1,4 @@
+addr: 10.52.37.14
 location:
   root: default
 labels:

host_vars/ceph-node02.yaml

@@ -1,3 +1,4 @@
+addr: 10.52.37.15
 location:
   root: default
 labels:

host_vars/ceph-node03.yaml

@@ -1,3 +1,4 @@
+addr: 10.52.37.16
 location:
   root: default
 labels:

inv.yaml

@@ -1,17 +1,17 @@
 ceph-nodes:
   hosts:
-    ceph01:
-      ansible_host: 10.50.42.23
-      ansible_user: mgr
-      ansible_ssh_pass: 12345
-      ansible_become_pass: 12345
-    ceph02:
-      ansible_host: 10.50.42.24
-      ansible_user: mgr
-      ansible_ssh_pass: 12345
-      ansible_become_pass: 12345
-    ceph03:
-      ansible_host: 10.50.42.25
-      ansible_user: mgr
-      ansible_ssh_pass: 12345
-      ansible_become_pass: 12345
+    ceph-node01:
+      ansible_host: 10.50.37.14
+      ansible_user: user
+      ansible_ssh_pass: 123456
+      ansible_become_pass: 123456
+    ceph-node02:
+      ansible_host: 10.50.37.15
+      ansible_user: user
+      ansible_ssh_pass: 123456
+      ansible_become_pass: 123456
+    ceph-node03:
+      ansible_host: 10.50.37.16
+      ansible_user: user
+      ansible_ssh_pass: 123456
+      ansible_become_pass: 123456

pre-deploy.yml

@@ -5,89 +5,52 @@
   gather_facts: true
   diff: true
   vars:
-    ceph_repo: "https://download.ceph.com/debian-reef"
+    ceph_repo: "http://nexus.test.repo.int.nt-com.ru/repository/ceph-squid/"
+    image_registry: "harbor.test.registry.int.nt-com.ru/proxy-quay"
+    prometheus_tag: "v3.0.0"
+    node_exporter_tag: "v1.8.2"
+    grafana_tag: "9.4.12"
+    alertmanager_tag: "v0.27.0"
     ntp_server: "ntp.int.nt-com.ru"
-    first_mon_ip: "10.63.1.19"
-    cluster_network: "10.63.0.0/24"
-    public_network: "10.63.1.0/24"
+    first_mon_ip: "10.52.37.14"
+#    cluster_network: "10.63.0.0/24"
+    public_network: "10.52.37.0/24"
     CEPH_RELEASE: '19.2.0'
     initial_dashboard_user: 'admin'
     initial_dashboard_password: 'qweqwe'
     allow_fqdn_hostname: 'true'
   tasks:
-    - name: download asc from ceph
-      get_url:
-        url: https://download.ceph.com/keys/release.asc
-        dest: /tmp
-
-    - name: Delete previously install ceph.gpg
-      file:
-        path: /usr/share/keyrings/ceph.gpg
-        state: absent
-
-    - name: import downloaded asc
-      shell:
-        chdir: /tmp
-        cmd: echo y | gpg -o /usr/share/keyrings/ceph.gpg --dearmor release.asc;
-
-    - name: Add ceph repo
-      apt_repository:
-        filename: ceph
-        repo: "deb [signed-by=/usr/share/keyrings/ceph.gpg] {{ ceph_repo }} {{ ansible_distribution_release }} main"
-        state: present
-
-#    - name: here bootstrap the the first node
-#      block:
-#      - name: download cephadm bootstrapper
-#        get_url:
-#          url: https://download.ceph.com/rpm-{{CEPH_RELEASE}}/el9/noarch/cephadm
-#          dest: /tmp
-#          mode: '777'
-
-    - name: Prepare customized ceph.conf
-      template:
-        src: ceph.conf.j2
-        dest: /root/ceph.conf
-
-    - name: install packages
-      apt:
-        pkg:
-          - ceph-common
-          - cephadm
-          - lvm2
-          - podman
-          - chrony
-          - mc
-        state: present
-
-    - name: Setup ntp client
-      template:
-        src: chrony.yaml.j2
-        dest: /etc/chrony/chrony.conf
-      notify:
-        - ceph-rollout:restart
-
-    - name: Templating cluster-spec
-      template:
-        src: cluster_spec.yml.j2
-        dest: /root/cluster_spec.yml
-        trim_blocks: true
-        lstrip_blocks: true
-      run_once: true
-      delegate_to: ceph01
-
-    - name: Flush restart handlers
-      meta: flush_handlers
+    - name: basic setups
+      import_tasks:
+        file: basics.yaml
 
     - name: running ceph bootstrap
       pause:
-        prompt: RUN THIS TO BOOTSTRAP THE CLUSTER "cephadm bootstrap --mon-ip {{ first_mon_ip }} --cluster-network {{ cluster_network }}
-          --initial-dashboard-user {{ initial_dashboard_user }} --initial-dashboard-password {{ initial_dashboard_password }} --apply-spec /root/cluster_spec.yml"
+        prompt: RUN THIS TO BOOTSTRAP THE CLUSTER "cephadm --image harbor.test.registry.int.nt-com.ru/proxy-quay/ceph/ceph:v19.2 bootstrap --mon-ip {{ first_mon_ip }} 
+          --initial-dashboard-user {{ initial_dashboard_user }} --initial-dashboard-password {{ initial_dashboard_password }}
+          --apply-spec /root/cluster_spec.yml --config /root/initial-ceph.conf"
+      when:
+        - cluster_network is undefined
+
+    - name: running ceph bootstrap
+      pause:
+        prompt: RUN THIS TO BOOTSTRAP THE CLUSTER "cephadm --image harbor.test.registry.int.nt-com.ru/proxy-quay/ceph/ceph:v19.2 bootstrap --mon-ip {{ first_mon_ip }} 
+          --cluster-network {{ cluster_network }}
+          --initial-dashboard-user {{ initial_dashboard_user }} --initial-dashboard-password {{ initial_dashboard_password }}
+          --apply-spec /root/cluster_spec.yml --config /root/initial-ceph.conf"
+      when:
+        - cluster_network is defined
+
 
   handlers:
-    - name: ceph-rollout:restart
+    - name: ceph-rollout:restart_chrony
       ansible.builtin.systemd_service:
         state: restarted
         name: chronyd
-      listen: "restart chronyd service"
+#      listen: "restart chronyd service"
+    - name: ceph-rollout:restart_sshd
+      ansible.builtin.systemd_service:
+        state: restarted
+        name: sshd
+#      listen: "restart sshd service"
 

templates/ceph.conf.j2

@@ -1,3 +1,5 @@
 [global]
 public_network = {{ public_network }}
-cluster_network = {{ cluster_network }}
+{% if cluster_network is defined -%}
+    cluster_network = {{ cluster_network }}
+{% endif %}

templates/chrony.yaml.j2

@@ -18,10 +18,10 @@ confdir /etc/chrony/conf.d
 # Approved by Ubuntu Technical Board on 2011-02-08.
 # See http://www.pool.ntp.org/join.html for more information.
 server {{ ntp_server }} iburst prefer
-pool ntp.ubuntu.com        iburst maxsources 4
-pool 0.ubuntu.pool.ntp.org iburst maxsources 1
-pool 1.ubuntu.pool.ntp.org iburst maxsources 1
-pool 2.ubuntu.pool.ntp.org iburst maxsources 2
+#pool ntp.ubuntu.com        iburst maxsources 4
+#pool 0.ubuntu.pool.ntp.org iburst maxsources 1
+#pool 1.ubuntu.pool.ntp.org iburst maxsources 1
+#pool 2.ubuntu.pool.ntp.org iburst maxsources 2
 
 # Use time sources from DHCP.
 sourcedir /run/chrony-dhcp

templates/cluster_spec.yml.j2

@@ -1,7 +1,7 @@
 {% macro nodes() %}
     {%- for _host in groups['all'] %}
 service_type: host
-addr: {{hostvars[_host]['ansible_host']}}
+addr: {{hostvars[_host]['addr']}}
 hostname: {{ _host }}
 location:
     {% for key, value in hostvars[_host]['location'].items() %}
@@ -31,5 +31,4 @@ placement:
   label: "osd"
 spec:
   data_devices:
-    all: true
----
+    all: true

templates/initial-ceph.j2

@@ -0,0 +1,6 @@
+[mgr]
+mgr/cephadm/container_image_prometheus = {{ image_registry }}/prometheus/prometheus:{{ prometheus_tag }}
+mgr/cephadm/container_image_node_exporter = {{ image_registry }}/prometheus/node-exporter:{{ node_exporter_tag }}
+mgr/cephadm/container_image_grafana = server-1:5000/ceph/ceph-grafana:{{ grafana_tag }}
+mgr/cephadm/container_image_alertmanager = server-1:5000/prometheus/alertmanager:{{ alertmanager_tag }}
+

templates/sshd_config.j2

@@ -0,0 +1,121 @@
+# This is the sshd server system-wide configuration file.  See
+# sshd_config(5) for more information.
+
+# This sshd was compiled with PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games
+
+# The strategy used for options in the default sshd_config shipped with
+# OpenSSH is to specify options with their default value where
+# possible, but leave them commented.  Uncommented options override the
+# default value.
+
+Include /etc/ssh/sshd_config.d/*.conf
+
+#Port 22
+#AddressFamily any
+#ListenAddress 0.0.0.0
+#ListenAddress ::
+
+#HostKey /etc/ssh/ssh_host_rsa_key
+#HostKey /etc/ssh/ssh_host_ecdsa_key
+#HostKey /etc/ssh/ssh_host_ed25519_key
+
+# Ciphers and keying
+#RekeyLimit default none
+
+# Logging
+#SyslogFacility AUTH
+#LogLevel INFO
+
+# Authentication:
+
+#LoginGraceTime 2m
+PermitRootLogin yes
+#StrictModes yes
+#MaxAuthTries 6
+#MaxSessions 10
+
+#PubkeyAuthentication yes
+
+# Expect .ssh/authorized_keys2 to be disregarded by default in future.
+#AuthorizedKeysFile     .ssh/authorized_keys .ssh/authorized_keys2
+
+#AuthorizedPrincipalsFile none
+
+#AuthorizedKeysCommand none
+#AuthorizedKeysCommandUser nobody
+
+# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts
+#HostbasedAuthentication no
+# Change to yes if you don't trust ~/.ssh/known_hosts for
+# HostbasedAuthentication
+#IgnoreUserKnownHosts no
+# Don't read the user's ~/.rhosts and ~/.shosts files
+#IgnoreRhosts yes
+
+# To disable tunneled clear text passwords, change to no here!
+#PasswordAuthentication yes
+#PermitEmptyPasswords no
+
+# Change to yes to enable challenge-response passwords (beware issues with
+# some PAM modules and threads)
+KbdInteractiveAuthentication no
+
+# Kerberos options
+#KerberosAuthentication no
+#KerberosOrLocalPasswd yes
+#KerberosTicketCleanup yes
+#KerberosGetAFSToken no
+
+# GSSAPI options
+#GSSAPIAuthentication no
+#GSSAPICleanupCredentials yes
+#GSSAPIStrictAcceptorCheck yes
+#GSSAPIKeyExchange no
+
+# Set this to 'yes' to enable PAM authentication, account processing,
+# and session processing. If this is enabled, PAM authentication will
+# be allowed through the KbdInteractiveAuthentication and
+# PasswordAuthentication.  Depending on your PAM configuration,
+# PAM authentication via KbdInteractiveAuthentication may bypass
+# the setting of "PermitRootLogin without-password".
+# If you just want the PAM account and session checks to run without
+# PAM authentication, then enable this but set PasswordAuthentication
+# and KbdInteractiveAuthentication to 'no'.
+UsePAM yes
+
+#AllowAgentForwarding yes
+#AllowTcpForwarding yes
+#GatewayPorts no
+X11Forwarding yes
+#X11DisplayOffset 10
+#X11UseLocalhost yes
+#PermitTTY yes
+PrintMotd no
+#PrintLastLog yes
+#TCPKeepAlive yes
+#PermitUserEnvironment no
+#Compression delayed
+#ClientAliveInterval 0
+#ClientAliveCountMax 3
+#UseDNS no
+#PidFile /run/sshd.pid
+#MaxStartups 10:30:100
+#PermitTunnel no
+#ChrootDirectory none
+#VersionAddendum none
+
+# no default banner path
+#Banner none
+
+# Allow client to pass locale environment variables
+AcceptEnv LANG LC_*
+
+# override default of no subsystems
+Subsystem       sftp    /usr/lib/openssh/sftp-server
+
+# Example of overriding settings on a per-user basis
+#Match User anoncvs
+#       X11Forwarding no
+#       AllowTcpForwarding no
+#       PermitTTY no
+#       ForceCommand cvs server