deep_scrub_choked

docs/.gitignore

@@ -0,0 +1 @@
+build

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = ceph-ansible
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/source/conf.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+#
+# ceph-ansible documentation build configuration file, created by
+# sphinx-quickstart on Wed Apr  5 11:55:38 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The root toctree document.
+root_doc = 'glossary'
+
+# General information about the project.
+project = u'ceph-ansible'
+copyright = u'2017-2018, Ceph team and individual contributors'
+author = u'Ceph team and individual contributors'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = u''
+# The full version, including alpha/beta/rc tags.
+release = u''
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ceph-ansibledoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (root_doc, 'ceph-ansible.tex', u'ceph-ansible Documentation',
+     u'Ceph team and individual contributors', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (root_doc, 'ceph-ansible', u'ceph-ansible Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (root_doc, 'ceph-ansible', u'ceph-ansible Documentation',
+     author, 'ceph-ansible', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+master_doc = 'index'

docs/source/day-2/osds.rst

@@ -0,0 +1,51 @@
+Adding/Removing OSD(s) after a cluster is deployed is a common operation that should be straight-forward to achieve.
+
+
+Adding osd(s)
+-------------
+
+Adding new OSD(s) on an existing host or adding a new OSD node can be achieved by running the main playbook with the ``--limit`` ansible option.
+You basically need to update your host_vars/group_vars with the new hardware and/or the inventory host file with the new osd nodes being added.
+
+The command used would be like following:
+
+``ansible-playbook -vv -i <your-inventory> site-container.yml --limit <node>``
+
+example:
+
+.. code-block:: shell
+
+   $ cat hosts
+   [mons]
+   mon-node-1
+   mon-node-2
+   mon-node-3
+
+   [mgrs]
+   mon-node-1
+   mon-node-2
+   mon-node-3
+
+   [osds]
+   osd-node-1
+   osd-node-2
+   osd-node-3
+   osd-node-99
+   
+   $ ansible-playbook -vv -i hosts site-container.yml --limit osd-node-99
+
+
+Shrinking osd(s)
+----------------
+
+Shrinking OSDs can be done by using the shrink-osd.yml playbook provided in ``infrastructure-playbooks`` directory.
+
+The variable ``osd_to_kill`` is a comma separated list of OSD IDs which must be passed to the playbook (passing it as an extra var is the easiest way).
+
+The playbook will shrink all osds passed in ``osd_to_kill`` serially.
+
+example:
+
+.. code-block:: shell
+
+   $ ansible-playbook -vv -i hosts infrastructure-playbooks/shrink-osd.yml -e osd_to_kill=1,2,3

docs/source/day-2/purge.rst

@@ -0,0 +1,15 @@
+Purging the cluster
+-------------------
+
+ceph-ansible provides two playbooks in ``infrastructure-playbooks`` for purging a Ceph cluster: ``purge-cluster.yml`` and ``purge-container-cluster.yml``.
+
+The names are pretty self-explanatory, ``purge-cluster.yml`` is intended to purge a non-containerized cluster whereas ``purge-container-cluster.yml`` is to purge a containerized cluster.
+
+example:
+
+.. code-block:: shell
+
+   $ ansible-playbook -vv -i hosts infrastructure-playbooks/purge-container-cluster.yml
+
+.. note::
+   These playbooks aren't intended to be run with the ``--limit`` option.

docs/source/day-2/upgrade.rst

@@ -0,0 +1,17 @@
+Upgrading the ceph cluster
+--------------------------
+
+ceph-ansible provides a playbook in ``infrastructure-playbooks`` for upgrading a Ceph cluster: ``rolling_update.yml``.
+
+This playbook could be used for both minor upgrades (X.Y to X.Z) or major upgrades (X to Y).
+
+Before running a major upgrade you need to update the ceph-ansible version first.
+
+example:
+
+.. code-block:: shell
+
+   $ ansible-playbook -vv -i hosts infrastructure-playbooks/rolling_update.yml
+
+.. note::
+   This playbook isn't intended to be run with the ``--limit`` ansible option.

docs/source/dev/index.rst

@@ -0,0 +1,102 @@
+Contribution Guidelines
+=======================
+
+The repository centralises all the Ansible roles. The roles are all part of the Ansible Galaxy.
+
+We love contribution and we love giving visibility to our contributors, this is why all the **commits must be signed-off**.
+
+Mailing list
+------------
+
+Please register the mailing list at http://lists.ceph.com/listinfo.cgi/ceph-ansible-ceph.com.
+
+IRC
+---
+
+Feel free to join us in the channel ``#ceph-ansible`` of the OFTC servers (https://www.oftc.net).
+
+GitHub
+------
+
+The main GitHub account for the project is at https://github.com/ceph/ceph-ansible/.
+
+Submit a patch
+--------------
+
+To start contributing just do:
+
+.. code-block:: console
+
+   $ git checkout -b my-working-branch
+   $ # do your changes #
+   $ git add -p
+
+If your change impacts a variable file in a role such as ``roles/ceph-common/defaults/main.yml``, you need to generate a ``group_vars`` file:
+
+.. code-block:: console
+
+   $ ./generate_group_vars_sample.sh
+
+You are finally ready to push your changes on GitHub:
+
+.. code-block:: console
+
+   $ git commit -s
+   $ git push origin my-working-branch
+
+Worked on a change and you don't want to resend a commit for a syntax fix?
+
+.. code-block:: console
+
+   $ # do your syntax change #
+   $ git commit --amend
+   $ git push -f origin my-working-branch
+
+Pull Request Testing
+--------------------
+
+Pull request testing is handled by Jenkins. All test must pass before your pull request will be merged.
+
+All of tests that are running are listed in the GitHub UI and will list their current status.
+
+If a test fails and you'd like to rerun it, comment on your pull request in the following format:
+
+.. code-block:: none
+
+   jenkins test $scenario_name
+
+For example:
+
+.. code-block:: none
+
+   jenkins test centos-non_container-all_daemons
+
+Backporting changes
+-------------------
+
+If a change should be backported to a ``stable-*`` Git branch:
+
+- Mark your pull request with the GitHub label "Backport" so we don't lose track of it.
+- Fetch the latest updates into your clone: ``git fetch``
+- Determine the latest available stable branch:
+  ``git branch -r --list "origin/stable-[0-9].[0-9]" | sort -r | sed 1q``
+- Create a new local branch for your pull request, based on the stable branch:
+  ``git checkout --no-track -b my-backported-change origin/stable-5.0``
+- Cherry-pick your change: ``git cherry-pick -x (your-sha1)``
+- Create a new pull request against the ``stable-5.0`` branch.
+- Ensure that your pull request's title has the prefix "backport:", so it's clear
+  to reviewers what this is about.
+- Add a comment in your backport pull request linking to the original (main) pull request.
+
+All changes to the stable branches should land in main first, so we avoid
+regressions.
+
+Once this is done, one of the project maintainers will tag the tip of the
+stable branch with your change. For example:
+
+.. code-block:: console
+
+   $ git checkout stable-5.0
+   $ git pull --ff-only
+   $ git tag v5.0.12
+   $ git push origin v5.0.12

docs/source/glossary.rst

@@ -0,0 +1,9 @@
+Glossary
+========
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Contents:
+
+   index
+   testing/glossary

docs/source/index.rst

@@ -0,0 +1,339 @@
+============
+ceph-ansible
+============
+
+Ansible playbooks for Ceph, the distributed filesystem.
+
+
+Installation
+============
+
+GitHub
+------
+
+You can install directly from the source on GitHub by following these steps:
+
+- Clone the repository:
+
+  .. code-block:: console
+
+     $ git clone https://github.com/ceph/ceph-ansible.git
+
+- Next, you must decide which branch of ``ceph-ansible`` you wish to use. There
+  are stable branches to choose from or you could use the main branch:
+
+  .. code-block:: console
+
+     $ git checkout $branch
+
+- Next, use pip and the provided requirements.txt to install Ansible and other
+  needed Python libraries:
+
+  .. code-block:: console
+
+     $ pip install -r requirements.txt
+
+.. _ansible-on-rhel-family:
+
+Ansible on RHEL and CentOS
+--------------------------
+
+You can acquire Ansible on RHEL and CentOS by installing from `Ansible channel <https://access.redhat.com/articles/3174981>`_.
+
+On RHEL:
+
+.. code-block:: console
+
+   $ subscription-manager repos --enable=rhel-7-server-ansible-2-rpms
+
+(CentOS does not use subscription-manager and already has "Extras" enabled by default.)
+
+.. code-block:: console
+
+   $ sudo yum install ansible
+
+Ansible on Ubuntu
+-----------------
+
+You can acquire Ansible on Ubuntu by using the `Ansible PPA <https://launchpad.net/~ansible/+archive/ubuntu/ansible>`_.
+
+.. code-block:: console
+
+   $ sudo add-apt-repository ppa:ansible/ansible
+   $ sudo apt update
+   $ sudo apt install ansible
+
+Ansible collections
+-------------------
+
+In order to install third-party collections that are required for ceph-ansible,
+please run:
+
+.. code-block:: console
+
+   $ ansible-galaxy install -r requirements.yml
+
+
+Releases
+========
+
+The following branches should be used depending on your requirements. The ``stable-*``
+branches have been QE tested and sometimes receive backport fixes throughout their lifecycle.
+The ``main`` branch should be considered experimental and used with caution.
+
+- ``stable-3.0`` Supports Ceph versions ``jewel`` and ``luminous``. This branch requires Ansible version ``2.4``.
+
+- ``stable-3.1`` Supports Ceph versions ``luminous`` and ``mimic``. This branch requires Ansible version ``2.4``.
+
+- ``stable-3.2`` Supports Ceph versions ``luminous`` and ``mimic``. This branch requires Ansible version ``2.6``.
+
+- ``stable-4.0`` Supports Ceph version ``nautilus``. This branch requires Ansible version ``2.9``.
+
+- ``stable-5.0`` Supports Ceph version ``octopus``. This branch requires Ansible version ``2.9``.
+
+- ``stable-6.0`` Supports Ceph version ``pacific``. This branch requires Ansible version ``2.10``.
+
+- ``stable-7.0`` Supports Ceph version ``quincy``. This branch requires Ansible version ``2.15``.
+
+- ``main`` Supports the main (devel) branch of Ceph. This branch requires Ansible version ``2.15`` or ``2.16``.
+
+.. NOTE:: ``stable-3.0`` and ``stable-3.1`` branches of ceph-ansible are deprecated and no longer maintained.
+
+Configuration and Usage
+=======================
+
+This project assumes you have a basic knowledge of how Ansible works and have already prepared your hosts for
+configuration by Ansible.
+
+After you've cloned the ``ceph-ansible`` repository, selected your branch and installed Ansible then you'll need to create
+your inventory file, playbook and configuration for your Ceph cluster.
+
+Inventory
+---------
+
+The Ansible inventory file defines the hosts in your cluster and what roles each host plays in your Ceph cluster. The default
+location for an inventory file is ``/etc/ansible/hosts`` but this file can be placed anywhere and used with the ``-i`` flag of
+``ansible-playbook``.
+
+An example inventory file would look like:
+
+.. code-block:: ini
+
+   [mons]
+   mon1
+   mon2
+   mon3
+
+   [osds]
+   osd1
+   osd2
+   osd3
+
+.. note::
+
+   For more information on Ansible inventories please refer to the Ansible documentation: http://docs.ansible.com/ansible/latest/intro_inventory.html
+
+Playbook
+--------
+
+You must have a playbook to pass to the ``ansible-playbook`` command when deploying your cluster. There is a sample playbook at the root of the ``ceph-ansible``
+project called ``site.yml.sample``. This playbook should work fine for most usages, but it does include by default every daemon group which might not be
+appropriate for your cluster setup. Perform the following steps to prepare your playbook:
+
+- Rename the sample playbook: ``mv site.yml.sample site.yml``
+
+- Modify the playbook as necessary for the requirements of your cluster
+
+.. note::
+
+   It's important the playbook you use is placed at the root of the ``ceph-ansible`` project. This is how Ansible will be able to find the roles that
+   ``ceph-ansible`` provides.
+
+Configuration Validation
+------------------------
+
+The ``ceph-ansible`` project provides config validation through the ``ceph-validate`` role. If you are using one of the provided playbooks this role will
+be run early in the deployment as to ensure you've given ``ceph-ansible`` the correct config. This check is only making sure that you've provided the
+proper config settings for your cluster, not that the values in them will produce a healthy cluster. For example, if you give an incorrect address for
+``monitor_address`` then the mon will still fail to join the cluster.
+
+An example of a validation failure might look like:
+
+.. code-block:: console
+
+   TASK [ceph-validate : validate provided configuration] *************************
+   task path: /Users/andrewschoen/dev/ceph-ansible/roles/ceph-validate/tasks/main.yml:3
+   Wednesday 02 May 2018  13:48:16 -0500 (0:00:06.984)       0:00:18.803 *********
+    [ERROR]: [mon0] Validation failed for variable: osd_objectstore
+
+    [ERROR]: [mon0] Given value for osd_objectstore: foo
+
+    [ERROR]: [mon0] Reason: osd_objectstore must be either 'bluestore' or 'filestore'
+
+    fatal: [mon0]: FAILED! => {
+        "changed": false
+        }
+
+Supported Validation
+^^^^^^^^^^^^^^^^^^^^
+
+The ``ceph-validate`` role currently supports validation of the proper config for the following
+osd scenarios:
+
+- ``collocated``
+- ``non-collocated``
+- ``lvm``
+
+The following install options are also validated by the ``ceph-validate`` role:
+
+- ``ceph_origin`` set to ``distro``
+- ``ceph_origin`` set to ``repository``
+- ``ceph_origin`` set to ``local``
+- ``ceph_repository`` set to ``dev``
+- ``ceph_repository`` set to ``community``
+
+
+Installation methods
+--------------------
+
+Ceph can be installed through several methods.
+
+.. toctree::
+   :maxdepth: 1
+
+   installation/methods
+
+Configuration
+-------------
+
+The configuration for your Ceph cluster will be set by the use of ansible variables that ``ceph-ansible`` provides. All of these options and their default
+values are defined in the ``group_vars/`` directory at the root of the ``ceph-ansible`` project. Ansible will use configuration in a ``group_vars/`` directory
+that is relative to your inventory file or your playbook. Inside of the ``group_vars/`` directory there are many sample Ansible configuration files that relate
+to each of the Ceph daemon groups by their filename. For example, the ``osds.yml.sample`` contains all the default configuration for the OSD daemons. The ``all.yml.sample``
+file is a special ``group_vars`` file that applies to all hosts in your cluster.
+
+.. note::
+
+   For more information on setting group or host specific configuration refer to the Ansible documentation: http://docs.ansible.com/ansible/latest/intro_inventory.html#splitting-out-host-and-group-specific-data
+
+At the most basic level you must tell ``ceph-ansible`` what version of Ceph you wish to install, the method of installation, your clusters network settings and
+how you want your OSDs configured. To begin your configuration rename each file in ``group_vars/`` you wish to use so that it does not include the ``.sample``
+at the end of the filename, uncomment the options you wish to change and provide your own value.
+
+An example configuration that deploys the upstream ``octopus`` version of Ceph with lvm batch method would look like this in ``group_vars/all.yml``:
+
+.. code-block:: yaml
+
+   ceph_origin: repository
+   ceph_repository: community
+   public_network: "192.168.3.0/24"
+   cluster_network: "192.168.4.0/24"
+   devices:
+     - '/dev/sda'
+     - '/dev/sdb'
+
+The following config options are required to be changed on all installations but there could be other required options depending on your OSD scenario
+selection or other aspects of your cluster.
+
+- ``ceph_origin``
+- ``public_network``
+
+
+When deploying RGW instance(s) you are required to set the ``radosgw_interface`` or ``radosgw_address`` config option.
+
+``ceph.conf`` Configuration File
+---------------------------------
+
+The supported method for defining your ``ceph.conf`` is to use the ``ceph_conf_overrides`` variable. This allows you to specify configuration options using
+an INI format. This variable can be used to override sections already defined in ``ceph.conf`` (see: ``roles/ceph-config/templates/ceph.conf.j2``) or to provide
+new configuration options.
+
+The following sections in ``ceph.conf`` are supported:
+
+* ``[global]``
+* ``[mon]``
+* ``[osd]``
+* ``[mds]``
+* ``[client.rgw.{instance_name}]``
+
+An example:
+
+.. code-block:: yaml
+
+   ceph_conf_overrides:
+      global:
+        foo: 1234
+        bar: 5678
+      osd:
+        osd_mkfs_type: ext4
+
+.. note::
+
+   We will no longer accept pull requests that modify the ``ceph.conf`` template unless it helps the deployment. For simple configuration tweaks
+   please use the ``ceph_conf_overrides`` variable.
+
+Full documentation for configuring each of the Ceph daemon types are in the following sections.
+
+OSD Configuration
+-----------------
+
+OSD configuration was used to be set by selecting an OSD scenario and providing the configuration needed for
+that scenario. As of nautilus in stable-4.0, the only scenarios available is ``lvm``.
+
+.. toctree::
+   :maxdepth: 1
+
+   osds/scenarios
+
+Day-2 Operations
+----------------
+
+ceph-ansible provides a set of playbook in ``infrastructure-playbooks`` directory in order to perform some basic day-2 operations.
+
+.. toctree::
+   :maxdepth: 1
+
+   day-2/osds
+   day-2/purge
+   day-2/upgrade
+
+RBD Mirroring
+-------------
+
+Ceph-ansible provides the role ``ceph-rbd-mirror`` that can setup an RBD mirror replication.
+
+.. toctree::
+   :maxdepth: 1
+
+   rbdmirror/index
+
+Contribution
+============
+
+See the following section for guidelines on how to contribute to ``ceph-ansible``.
+
+.. toctree::
+   :maxdepth: 1
+
+   dev/index
+
+Testing
+=======
+
+Documentation for writing functional testing scenarios for ``ceph-ansible``.
+
+* :doc:`Testing with ceph-ansible <testing/index>`
+* :doc:`Glossary <testing/glossary>`
+
+Demos
+=====
+
+Vagrant Demo
+------------
+
+Deployment from scratch on vagrant machines: https://youtu.be/E8-96NamLDo
+
+Bare metal demo
+---------------
+
+Deployment from scratch on bare metal machines: https://youtu.be/dv_PEp9qAqg

docs/source/installation/containerized.rst

@@ -0,0 +1,64 @@
+Containerized deployment
+========================
+
+Ceph-ansible supports docker and podman only in order to deploy Ceph in a containerized context.
+
+Configuration and Usage
+-----------------------
+
+To deploy ceph in containers, you will need to set the ``containerized_deployment`` variable to ``true`` and use the site-container.yml.sample playbook.
+
+.. code-block:: yaml
+
+   containerized_deployment: true
+
+The ``ceph_origin`` and ``ceph_repository`` variables aren't needed anymore in containerized deployment and are ignored.
+
+.. code-block:: console
+
+   $ ansible-playbook site-container.yml.sample
+
+.. note::
+
+   The infrastructure playbooks are working for both non containerized and containerized deployment.
+
+Custom container image
+----------------------
+
+You can configure your own container register, image and tag by using the ``ceph_docker_registry``, ``ceph_docker_image`` and ``ceph_docker_image_tag`` variables.
+
+.. code-block:: yaml
+
+   ceph_docker_registry: quay.io
+   ceph_docker_image: ceph/ceph
+   ceph_docker_image_tag: v19
+
+.. note::
+
+   ``ceph_docker_image`` should have both image namespace and image name concatenated and separated by a slash character.
+
+   ``ceph_docker_image_tag`` should be set to a fixed tag, not to any "latest" tags unless you know what you are doing. Using a "latest" tag
+   might make the playbook restart all the daemons deployed in your cluster since these tags are intended to be updated periodically.
+
+Container registry authentication
+---------------------------------
+
+When using a container registry with authentication then you need to set the ``ceph_docker_registry_auth`` variable to ``true`` and provide the credentials via the
+``ceph_docker_registry_username`` and ``ceph_docker_registry_password`` variables
+
+.. code-block:: yaml
+
+   ceph_docker_registry_auth: true
+   ceph_docker_registry_username: foo
+   ceph_docker_registry_password: bar
+
+Container registry behind a proxy
+---------------------------------
+
+When using a container registry reachable via a http(s) proxy then you need to set the ``ceph_docker_http_proxy`` and/or ``ceph_docker_https_proxy`` variables. If you need
+to exclude some host for the proxy configuration to can use the ``ceph_docker_no_proxy`` variable.
+
+.. code-block:: yaml
+
+   ceph_docker_http_proxy: http://192.168.42.100:8080
+   ceph_docker_https_proxy: https://192.168.42.100:8080

docs/source/installation/methods.rst

@@ -0,0 +1,12 @@
+Installation methods
+====================
+
+ceph-ansible can deploy Ceph either in a non-containerized context (via packages) or in a containerized context using ceph-container images.
+
+.. toctree::
+   :maxdepth: 1
+
+   non-containerized
+   containerized
+
+The difference here is that you don't have the rbd command on the host when using the containerized deployment so everything related to ceph needs to be executed within a container. So in the case there is software like e.g. Open Nebula which requires that the rbd command is accessible directly on the host (non-containerized) then you have to install the rbd command by yourself on those servers outside of containers (or make sure that this software somehow runs within containers as well and that it can access rbd).

docs/source/installation/non-containerized.rst

@@ -0,0 +1,58 @@
+Non containerized deployment
+============================
+
+The following are all of the available options for the installing Ceph through different channels.
+
+We support 3 main installation methods, all managed by the ``ceph_origin`` variable:
+
+- ``repository``: means that you will get Ceph installed through a new repository. Later below choose between ``community`` or ``dev``. These options will be exposed through the ``ceph_repository`` variable.
+- ``distro``: means that no separate repo file will be added and you will get whatever version of Ceph is included in your Linux distro.
+- ``local``: means that the Ceph binaries will be copied over from the local machine (not well tested, use at your own risk)
+
+Origin: Repository
+------------------
+
+If ``ceph_origin`` is set to ``repository``, you now have the choice between a couple of repositories controlled by the ``ceph_repository`` option:
+
+- ``community``: fetches packages from http://download.ceph.com, the official community Ceph repositories
+- ``dev``: fetches packages from shaman, a gitbuilder based package system
+- ``uca``: fetches packages from Ubuntu Cloud Archive
+- ``custom``: fetches packages from a specific repository
+
+Community repository
+~~~~~~~~~~~~~~~~~~~~
+
+If ``ceph_repository`` is set to ``community``, packages you will be by default installed from http://download.ceph.com, this can be changed by tweaking ``ceph_mirror``.
+
+
+UCA repository
+~~~~~~~~~~~~~~
+
+If ``ceph_repository`` is set to ``uca``, packages you will be by default installed from http://ubuntu-cloud.archive.canonical.com/ubuntu, this can be changed by tweaking ``ceph_stable_repo_uca``.
+You can also decide which OpenStack version the Ceph packages should come from by tweaking ``ceph_stable_openstack_release_uca``.
+For example, ``ceph_stable_openstack_release_uca: queens``.
+
+Dev repository
+~~~~~~~~~~~~~~
+
+If ``ceph_repository`` is set to ``dev``, packages you will be by default installed from https://shaman.ceph.com/, this can not be tweaked.
+You can obviously decide which branch to install with the help of  ``ceph_dev_branch`` (defaults to 'main').
+Additionally, you can specify a SHA1 with ``ceph_dev_sha1``, defaults to 'latest' (as in latest built).
+
+Custom repository
+~~~~~~~~~~~~~~~~~
+
+If ``ceph_repository`` is set to ``custom``, packages you will be by default installed from a desired repository.
+This repository is specified with ``ceph_custom_repo``, e.g: ``ceph_custom_repo: https://server.domain.com/ceph-custom-repo``.
+
+
+Origin: Distro
+--------------
+
+If ``ceph_origin`` is set to ``distro``, no separate repo file will be added and you will get whatever version of Ceph is included in your Linux distro.
+
+
+Origin: Local
+-------------
+
+If ``ceph_origin`` is set to ``local``, the ceph binaries will be copied over from the local machine (not well tested, use at your own risk)

docs/source/osds/scenarios.rst

@@ -0,0 +1,221 @@
+OSD Scenario
+============
+
+As of stable-4.0, the following scenarios are not supported anymore since they are associated to ``ceph-disk``:
+
+* `collocated`
+* `non-collocated`
+
+Since the Ceph luminous release, it is preferred to use the :ref:`lvm scenario
+<osd_scenario_lvm>` that uses the ``ceph-volume`` provisioning tool. Any other
+scenario will cause deprecation warnings.
+
+``ceph-disk`` was deprecated during the ceph-ansible 3.2 cycle and has been removed entirely from Ceph itself in the Nautilus version.
+At present (starting from stable-4.0), there is only one scenario, which defaults to ``lvm``, see:
+
+* :ref:`lvm <osd_scenario_lvm>`
+
+So there is no need to configure ``osd_scenario`` anymore, it defaults to ``lvm``.
+
+The ``lvm`` scenario mentioned above support both containerized and non-containerized cluster.
+As a reminder, deploying a containerized cluster can be done by setting ``containerized_deployment``
+to ``True``.
+
+If you want to skip OSD creation during a ``ceph-ansible run``
+(e.g. because you have already provisioned your OSDs but disk IDs have
+changed), you can skip the ``prepare_osd`` tag i.e. by specifying
+``--skip-tags prepare_osd`` on the ``ansible-playbook`` command line.
+
+.. _osd_scenario_lvm:
+
+lvm
+---
+
+This OSD scenario uses ``ceph-volume`` to create OSDs, primarily using LVM, and
+is only available when the Ceph release is luminous or newer.
+It is automatically enabled.
+
+Other (optional) supported settings:
+
+- ``dmcrypt``: Enable Ceph's encryption on OSDs using ``dmcrypt``.
+    Defaults to ``false`` if unset.
+
+- ``osds_per_device``: Provision more than 1 OSD (the default if unset) per device.
+
+
+Simple configuration
+^^^^^^^^^^^^^^^^^^^^
+
+With this approach, most of the decisions on how devices are configured to
+provision an OSD are made by the Ceph tooling (``ceph-volume lvm batch`` in
+this case).  There is almost no room to modify how the OSD is composed given an
+input of devices.
+
+To use this configuration, the ``devices`` option must be populated with the
+raw device paths that will be used to provision the OSDs.
+
+
+.. note:: Raw devices must be "clean", without a gpt partition table, or
+          logical volumes present.
+
+
+For example, for a node that has ``/dev/sda`` and ``/dev/sdb`` intended for
+Ceph usage, the configuration would be:
+
+
+.. code-block:: yaml
+
+   devices:
+     - /dev/sda
+     - /dev/sdb
+
+In the above case, if both devices are spinning drives, 2 OSDs would be
+created, each with its own collocated journal.
+
+Other provisioning strategies are possible, by mixing spinning and solid state
+devices, for example:
+
+.. code-block:: yaml
+
+   devices:
+     - /dev/sda
+     - /dev/sdb
+     - /dev/nvme0n1
+
+Similar to the initial example, this would end up producing 2 OSDs, but data
+would be placed on the slower spinning drives (``/dev/sda``, and ``/dev/sdb``)
+and journals would be placed on the faster solid state device ``/dev/nvme0n1``.
+The ``ceph-volume`` tool describes this in detail in
+`the "batch" subcommand section <https://docs.ceph.com/en/latest/ceph-volume/lvm/batch/>`_
+
+This option can also be used with ``osd_auto_discovery``, meaning that you do not need to populate
+``devices`` directly and any appropriate devices found by ansible will be used instead.
+
+.. code-block:: yaml
+
+   osd_auto_discovery: true
+
+Other (optional) supported settings:
+
+- ``crush_device_class``: Sets the CRUSH device class for all OSDs created with this
+  method (it is not possible to have a per-OSD CRUSH device class using the *simple*
+  configuration approach). Values *must be* a string, like
+  ``crush_device_class: "ssd"``
+
+
+Advanced configuration
+^^^^^^^^^^^^^^^^^^^^^^
+
+This configuration is useful when more granular control is wanted when setting
+up devices and how they should be arranged to provision an OSD. It requires an
+existing setup of volume groups and logical volumes (``ceph-volume`` will **not**
+create these).
+
+To use this configuration, the ``lvm_volumes`` option must be populated with
+logical volumes and volume groups. Additionally, absolute paths to partitions
+*can* be used for ``journal``, ``block.db``, and ``block.wal``.
+
+.. note:: This configuration uses ``ceph-volume lvm create`` to provision OSDs
+
+Supported ``lvm_volumes`` configuration settings:
+
+- ``data``: The logical volume name or full path to a raw device (an LV will be
+  created using 100% of the raw device)
+
+- ``data_vg``: The volume group name, **required** if ``data`` is a logical volume.
+
+- ``crush_device_class``: CRUSH device class name for the resulting OSD, allows
+  setting set the device class for each OSD, unlike the global ``crush_device_class``
+  that sets them for all OSDs.
+
+.. note:: If you wish to set the ``crush_device_class`` for the OSDs
+          when using ``devices`` you must set it using the global ``crush_device_class``
+          option as shown above. There is no way to define a specific CRUSH device class
+          per OSD when using ``devices`` like there is for ``lvm_volumes``.
+
+
+.. warning:: Each entry must be unique, duplicate values are not allowed
+
+
+``bluestore`` objectstore variables:
+
+- ``db``: The logical volume name or full path to a partition.
+
+- ``db_vg``: The volume group name, **required** if ``db`` is a logical volume.
+
+- ``wal``: The logical volume name or full path to a partition.
+
+- ``wal_vg``: The volume group name, **required** if ``wal`` is a logical volume.
+
+
+.. note:: These ``bluestore`` variables are optional optimizations. Bluestore's
+          ``db`` and ``wal`` will only benefit from faster devices. It is possible to
+          create a bluestore OSD with a single raw device.
+
+.. warning:: Each entry must be unique, duplicate values are not allowed
+
+
+``bluestore`` example using raw devices:
+
+.. code-block:: yaml
+
+   osd_objectstore: bluestore
+   lvm_volumes:
+     - data: /dev/sda
+     - data: /dev/sdb
+
+.. note:: Volume groups and logical volumes will be created in this case,
+          utilizing 100% of the devices.
+
+``bluestore`` example with logical volumes:
+
+.. code-block:: yaml
+
+   osd_objectstore: bluestore
+   lvm_volumes:
+     - data: data-lv1
+       data_vg: data-vg1
+     - data: data-lv2
+       data_vg: data-vg2
+
+.. note:: Volume groups and logical volumes must exist.
+
+
+``bluestore`` example defining ``wal`` and ``db`` logical volumes:
+
+.. code-block:: yaml
+
+   osd_objectstore: bluestore
+   lvm_volumes:
+     - data: data-lv1
+       data_vg: data-vg1
+       db: db-lv1
+       db_vg: db-vg1
+       wal: wal-lv1
+       wal_vg: wal-vg1
+     - data: data-lv2
+       data_vg: data-vg2
+       db: db-lv2
+       db_vg: db-vg2
+       wal: wal-lv2
+       wal_vg: wal-vg2
+
+.. note:: Volume groups and logical volumes must exist.
+
+
+``filestore`` example with logical volumes:
+
+.. code-block:: yaml
+
+   osd_objectstore: filestore
+   lvm_volumes:
+     - data: data-lv1
+       data_vg: data-vg1
+       journal: journal-lv1
+       journal_vg: journal-vg1
+     - data: data-lv2
+       data_vg: data-vg2
+       journal: journal-lv2
+       journal_vg: journal-vg2
+
+.. note:: Volume groups and logical volumes must exist.

docs/source/rbdmirror/index.rst

@@ -0,0 +1,60 @@
+RBD Mirroring
+=============
+
+There's not so much to do from the primary cluster side in order to setup an RBD mirror replication.
+``ceph_rbd_mirror_configure`` has to be set to ``true`` to make ceph-ansible create the mirrored pool
+defined in ``ceph_rbd_mirror_pool`` and the keyring that is going to be used to add the rbd mirror peer.
+
+group_vars from the primary cluster:
+
+.. code-block:: yaml
+
+      ceph_rbd_mirror_configure: true
+      ceph_rbd_mirror_pool: rbd
+
+Optionnally, you can tell ceph-ansible to set the name and the secret of the keyring you want to create:
+
+.. code-block:: yaml
+
+      ceph_rbd_mirror_local_user: client.rbd-mirror-peer  # 'client.rbd-mirror-peer' is the default value.
+      ceph_rbd_mirror_local_user_secret: AQC+eM1iKKBXFBAAVpunJvqpkodHSYmljCFCnw==
+
+This secret will be needed to add the rbd mirror peer from the secondary cluster.
+If you do not enforce it as shown above, you can get it from a monitor by running the following command:
+``ceph auth get {{ ceph_rbd_mirror_local_user }}``
+
+
+.. code-block:: shell
+
+    $ sudo ceph auth get client.rbd-mirror-peer
+
+Once your variables are defined, you can run the playbook (you might want to run with --limit option):
+
+.. code-block:: shell
+
+    $ ansible-playbook -vv -i hosts site-container.yml --limit rbdmirror0
+
+
+The configuration of the rbd mirror replication strictly speaking is done on the secondary cluster.
+The rbd-mirror daemon pulls the data from the primary cluster. This is where the rbd mirror peer addition has to be done.
+The configuration is similar with what was done on the primary cluster, it just needs few additional variables.
+
+``ceph_rbd_mirror_remote_user`` : This user must match the name defined in the variable ``ceph_rbd_mirror_local_user`` from the primary cluster.
+``ceph_rbd_mirror_remote_mon_hosts`` : This must a comma separated list of the monitor addresses from the primary cluster.
+``ceph_rbd_mirror_remote_key`` : This must be the same value as the user (``{{ ceph_rbd_mirror_local_user }}``) keyring secret from the primary cluster.
+
+group_vars from the secondary cluster:
+
+.. code-block:: yaml
+
+   ceph_rbd_mirror_configure: true
+   ceph_rbd_mirror_pool: rbd
+   ceph_rbd_mirror_remote_user: client.rbd-mirror-peer  # This must match the value defined in {{ ceph_rbd_mirror_local_user }} on primary cluster.
+   ceph_rbd_mirror_remote_mon_hosts: 1.2.3.4
+   ceph_rbd_mirror_remote_key: AQC+eM1iKKBXFBAAVpunJvqpkodHSYmljCFCnw==  # This must match the secret of the registered keyring of the user defined in {{ ceph_rbd_mirror_local_user }} on primary cluster.
+
+Once you variables are defined, you can run the playbook (you might want to run with --limit option):
+
+.. code-block:: shell
+
+    $ ansible-playbook -vv -i hosts site-container.yml --limit rbdmirror0

docs/source/testing/development.rst

@@ -0,0 +1,4 @@
+.. _development:
+
+ceph-ansible testing for development
+====================================

docs/source/testing/glossary.rst

@@ -0,0 +1,14 @@
+Glossary
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   index
+   running.rst
+   development.rst
+   scenarios.rst
+   modifying.rst
+   layout.rst
+   tests.rst
+   tox.rst

docs/source/testing/index.rst

@@ -0,0 +1,38 @@
+.. _testing:
+
+Testing
+=======
+
+``ceph-ansible`` has the ability to test different scenarios (collocated journals
+or dmcrypt OSDs for example) in an isolated, repeatable, and easy way.
+
+These tests can run locally with VirtualBox or via libvirt if available, which
+removes the need to solely rely on a CI system like Jenkins to verify
+a behavior.
+
+* **Getting started:**
+
+  * :doc:`Running a Test Scenario <running>`
+  * :ref:`dependencies`
+
+* **Configuration and structure:**
+
+  * :ref:`layout`
+  * :ref:`test_files`
+  * :ref:`scenario_files`
+  * :ref:`scenario_wiring`
+
+* **Adding or modifying tests:**
+
+  * :ref:`test_conventions`
+  * :ref:`testinfra`
+
+* **Adding or modifying a scenario:**
+
+  * :ref:`scenario_conventions`
+  * :ref:`scenario_environment_configuration`
+  * :ref:`scenario_ansible_configuration`
+
+* **Custom/development repositories and packages:**
+
+  * :ref:`tox_environment_variables`

docs/source/testing/layout.rst

@@ -0,0 +1,60 @@
+.. _layout:
+
+Layout and conventions
+----------------------
+
+Test files and directories follow a few conventions, which makes it easy to
+create (or expect) certain interactions between tests and scenarios.
+
+All tests are in the ``tests`` directory. Scenarios are defined in
+``tests/functional/`` and use the following convention for directory
+structure:
+
+.. code-block:: none
+
+    tests/functional/<distro>/<distro version>/<scenario name>/
+
+For example: ``tests/functional/centos/7/journal-collocation``
+
+Within a test scenario there are a few files that define what that specific
+scenario needs for the tests, like how many OSD nodes or MON nodes. Tls
+
+At the very least, a scenario will need these files:
+
+* ``Vagrantfile``: must be symlinked from the root directory of the project
+* ``hosts``: An Ansible hosts file that defines the machines part of the
+  cluster
+* ``group_vars/all``: if any modifications are needed for deployment, this
+  would override them. Additionally, further customizations can be done. For
+  example, for OSDs that would mean adding ``group_vars/osds``
+* ``vagrant_variables.yml``: Defines the actual environment for the test, where
+  machines, networks, disks, linux distro/version, can be defined.
+
+
+.. _test_conventions:
+
+Conventions
+-----------
+
+Python test files (unlike scenarios) rely on paths to *map* where they belong. For
+example, a file that should only test monitor nodes would live in
+``ceph-ansible/tests/functional/tests/mon/``. Internally, the test runner
+(``py.test``) will *mark* these as tests that should run on a monitor only.
+Since the configuration of a scenario already defines what node has a given
+role, then it is easier for the system to only run tests that belong to
+a particular node type.
+
+The current convention is a bit manual, with initial path support for:
+
+* mon
+* osd
+* mds
+* rgw
+* journal_collocation
+* all/any (if none of the above are matched, then these are run on any host)
+
+
+.. _testinfra:
+
+``testinfra``
+-------------

docs/source/testing/modifying.rst

@@ -0,0 +1,4 @@
+.. _modifying:
+
+Modifying (or adding) tests
+===========================

docs/source/testing/running.rst

@@ -0,0 +1,169 @@
+.. _running_tests:
+
+Running Tests
+=============
+
+Although tests run continuously in CI, a lot of effort was put into making it
+easy to run in any environment, as long as a couple of requirements are met.
+
+
+.. _dependencies:
+
+Dependencies
+------------
+
+There are some Python dependencies, which are listed in a ``requirements.txt``
+file within the ``tests/`` directory. These are meant to be installed using
+Python install tools (pip in this case):
+
+.. code-block:: console
+
+   pip install -r tests/requirements.txt
+
+For virtualization, either libvirt or VirtualBox is needed (there is native
+support from the harness for both). This makes the test harness even more
+flexible as most platforms will be covered by either VirtualBox or libvirt.
+
+
+.. _running_a_scenario:
+
+Running a scenario
+------------------
+
+Tests are driven by ``tox``, a command line tool to run a matrix of tests defined in
+a configuration file (``tox.ini`` in this case at the root of the project).
+
+For a thorough description of a scenario see :ref:`test_scenarios`.
+
+To run a single scenario, make sure it is available (should be defined from
+``tox.ini``) by listing them:
+
+.. code-block:: console
+
+   tox -l
+
+In this example, we will use the ``luminous-ansible2.4-xenial_cluster`` one.  The
+harness defaults to ``VirtualBox`` as the backend, so if you have that
+installed in your system then this command should just work:
+
+.. code-block:: console
+
+   tox -e luminous-ansible2.4-xenial_cluster
+
+And for libvirt it would be:
+
+.. code-block:: console
+
+   tox -e luminous-ansible2.4-xenial_cluster -- --provider=libvirt
+
+.. warning::
+
+   Depending on the type of scenario and resources available, running
+   these tests locally in a personal computer can be very resource intensive.
+
+.. note::
+
+   Most test runs take between 20 and 40 minutes depending on system
+   resources
+
+The command should bring up the machines needed for the test, provision them
+with ``ceph-ansible``, run the tests, and tear the whole environment down at the
+end.
+
+
+The output would look something similar to this trimmed version:
+
+.. code-block:: console
+
+   luminous-ansible2.4-xenial_cluster create: /Users/alfredo/python/upstream/ceph-ansible/.tox/luminous-ansible2.4-xenial_cluster
+   luminous-ansible2.4-xenial_cluster installdeps: ansible==2.4.2, -r/Users/alfredo/python/upstream/ceph-ansible/tests/requirements.txt
+   luminous-ansible2.4-xenial_cluster runtests: commands[0] | vagrant up --no-provision --provider=virtualbox
+   Bringing machine 'client0' up with 'virtualbox' provider...
+   Bringing machine 'rgw0' up with 'virtualbox' provider...
+   Bringing machine 'mds0' up with 'virtualbox' provider...
+   Bringing machine 'mon0' up with 'virtualbox' provider...
+   Bringing machine 'mon1' up with 'virtualbox' provider...
+   Bringing machine 'mon2' up with 'virtualbox' provider...
+   Bringing machine 'osd0' up with 'virtualbox' provider...
+   ...
+
+
+After all the nodes are up, ``ceph-ansible`` will provision them, and run the
+playbook(s):
+
+.. code-block:: console
+
+   ...
+   PLAY RECAP *********************************************************************
+   client0                    : ok=4    changed=0    unreachable=0    failed=0
+   mds0                       : ok=4    changed=0    unreachable=0    failed=0
+  mon0                       : ok=4    changed=0    unreachable=0    failed=0
+   mon1                       : ok=4    changed=0    unreachable=0    failed=0
+   mon2                       : ok=4    changed=0    unreachable=0    failed=0
+   osd0                       : ok=4    changed=0    unreachable=0    failed=0
+   rgw0                       : ok=4    changed=0    unreachable=0    failed=0
+   ...
+
+
+Once the whole environment is all running the tests will be sent out to the
+hosts, with output similar to this:
+
+.. code-block:: console
+
+   luminous-ansible2.4-xenial_cluster runtests: commands[4] | testinfra -n 4 --sudo -v --connection=ansible --ansible-inventory=/Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster/hosts /Users/alfredo/python/upstream/ceph-ansible/tests/functional/tests
+   ============================ test session starts ===========================
+   platform darwin -- Python 2.7.8, pytest-3.0.7, py-1.4.33, pluggy-0.4.0 -- /Users/alfredo/python/upstream/ceph-ansible/.tox/luminous-ansible2.4-xenial_cluster/bin/python
+   cachedir: ../../../../.cache
+   rootdir: /Users/alfredo/python/upstream/ceph-ansible/tests, inifile: pytest.ini
+   plugins: testinfra-1.5.4, xdist-1.15.0
+   [gw0] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+   [gw1] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+   [gw2] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+   [gw3] darwin Python 2.7.8 cwd: /Users/alfredo/python/upstream/ceph-ansible/tests/functional/ubuntu/16.04/cluster
+   [gw0] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+   [gw1] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+   [gw2] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+   [gw3] Python 2.7.8 (v2.7.8:ee879c0ffa11, Jun 29 2014, 21:07:35)  -- [GCC 4.2.1 (Apple Inc. build 5666) (dot 3)]
+   gw0 [154] / gw1 [154] / gw2 [154] / gw3 [154]
+   scheduling tests via LoadScheduling
+
+   ../../../tests/test_install.py::TestInstall::test_ceph_dir_exists[ansible:/mon0]
+   ../../../tests/test_install.py::TestInstall::test_ceph_dir_is_a_directory[ansible:/mon0]
+   ../../../tests/test_install.py::TestInstall::test_ceph_conf_is_a_file[ansible:/mon0]
+   ../../../tests/test_install.py::TestInstall::test_ceph_dir_is_a_directory[ansible:/mon1]
+   [gw2] PASSED ../../../tests/test_install.py::TestCephConf::test_ceph_config_has_mon_host_line[ansible:/mon0]
+   ../../../tests/test_install.py::TestInstall::test_ceph_conf_exists[ansible:/mon1]
+   [gw3] PASSED ../../../tests/test_install.py::TestCephConf::test_mon_host_line_has_correct_value[ansible:/mon0]
+   ../../../tests/test_install.py::TestInstall::test_ceph_conf_is_a_file[ansible:/mon1]
+   [gw1] PASSED ../../../tests/test_install.py::TestInstall::test_ceph_command_exists[ansible:/mon1]
+   ../../../tests/test_install.py::TestCephConf::test_mon_host_line_has_correct_value[ansible:/mon1]
+   ...
+
+Finally the whole environment gets torn down:
+
+.. code-block:: console
+
+   luminous-ansible2.4-xenial_cluster runtests: commands[5] | vagrant destroy --force
+   ==> osd0: Forcing shutdown of VM...
+   ==> osd0: Destroying VM and associated drives...
+   ==> mon2: Forcing shutdown of VM...
+   ==> mon2: Destroying VM and associated drives...
+   ==> mon1: Forcing shutdown of VM...
+   ==> mon1: Destroying VM and associated drives...
+   ==> mon0: Forcing shutdown of VM...
+   ==> mon0: Destroying VM and associated drives...
+   ==> mds0: Forcing shutdown of VM...
+   ==> mds0: Destroying VM and associated drives...
+   ==> rgw0: Forcing shutdown of VM...
+   ==> rgw0: Destroying VM and associated drives...
+   ==> client0: Forcing shutdown of VM...
+   ==> client0: Destroying VM and associated drives...
+
+
+And a brief summary of the scenario(s) that ran is displayed:
+
+.. code-block:: console
+
+   ________________________________________________ summary _________________________________________________
+     luminous-ansible2.4-xenial_cluster: commands succeeded
+     congratulations :)

docs/source/testing/scenarios.rst

@@ -0,0 +1,211 @@
+.. _test_scenarios:
+
+Test Scenarios
+==============
+
+Scenarios are distinct environments that describe a Ceph deployment and
+configuration. Scenarios are isolated as well, and define what machines are
+needed aside from any ``ceph-ansible`` configuration.
+
+.. _scenario_files:
+
+Scenario Files
+==============
+
+The scenario is described in a ``vagrant_variables.yml`` file, which is
+consumed by ``Vagrant`` when bringing up an environment.
+
+This yaml file is loaded in the ``Vagrantfile`` so that the settings can be
+used to bring up the boxes and pass some configuration to ansible when running.
+
+.. note::
+
+   The basic layout of a scenario is covered in :ref:`layout`.
+   There are just a handful of required files, this is the most basic layout.
+
+There are just a handful of required files, these sections will cover the
+required (most basic) ones. Alternatively, other ``ceph-ansible`` files can be
+added to customize the behavior of a scenario deployment.
+
+
+.. _vagrant_variables:
+
+``vagrant_variables.yml``
+-------------------------
+
+There are a few sections in the ``vagrant_variables.yml`` file which are easy
+to follow (most of them are 1 line settings).
+
+* **docker**: (bool) Indicates if the scenario will deploy Docker daemons
+
+* **VMS**: (int) These integer values are just a count of how  many machines will be
+  needed. Each supported type is listed, defaulting to 0:
+
+  .. code-block:: yaml
+
+     mon_vms: 0
+     osd_vms: 0
+     mds_vms: 0
+     rgw_vms: 0
+     nfs_vms: 0
+     rbd_mirror_vms: 0
+     client_vms: 0
+     mgr_vms: 0
+
+  For a deployment that needs 1 MON and 1 OSD, the list would look like:
+
+  .. code-block:: yaml
+
+     mon_vms: 1
+     osd_vms: 1
+
+* **CEPH SOURCE**: (string) indicate whether a ``dev`` or ``stable`` release is
+  needed. A ``stable`` release will use the latest stable release of Ceph,
+  a ``dev`` will use ``shaman`` (http://shaman.ceph.com)
+
+* **SUBNETS**: These are used for configuring the network availability of each
+  server that will be booted as well as being used as configuration for
+  ``ceph-ansible`` (and eventually Ceph). The two values that are **required**:
+
+  .. code-block:: yaml
+
+     public_subnet: 192.168.13
+     cluster_subnet: 192.168.14
+
+* **MEMORY**: Memory requirements (in megabytes) for each server, e.g.
+  ``memory: 512``
+
+* **interfaces**: some vagrant boxes (and linux distros) set specific
+  interfaces. For Ubuntu releases older than Xenial it was common to have
+  ``eth1``, for CentOS and some Xenial boxes ``enp0s8`` is used. **However**
+  the public Vagrant boxes normalize the interface to ``eth1`` for all boxes,
+  making it easier to configure them with Ansible later.
+
+.. warning::
+
+   Do *not* change the interface from ``eth1`` unless absolutely
+   certain that is needed for a box. Some tests that depend on that
+   naming will fail.
+
+* **disks**: The disks that will be created for each machine, for most
+  environments ``/dev/sd*`` style of disks will work, like: ``[ '/dev/sda', '/dev/sdb' ]``
+
+* **vagrant_box**: We have published our own boxes to normalize what we test
+  against. These boxes are published in Atlas
+  (https://atlas.hashicorp.com/ceph/). Currently valid values are:
+  ``ceph/ubuntu-xenial``, and ``ceph/centos7``
+
+The following aren't usually changed/enabled for tests, since they don't have
+an impact, however they are documented here for general knowledge in case they
+are needed:
+
+* **ssh_private_key_path**: The path to the ``id_rsa`` (or other private SSH
+  key) that should be used to connect to these boxes.
+
+* **vagrant_sync_dir**: what should be "synced" (made available on the new
+  servers) from the host.
+
+* **vagrant_disable_synced_folder**: (bool) when disabled, it will make
+  booting machines faster because no files need to be synced over.
+
+* **os_tuning_params**: These are passed onto ``ceph-ansible`` as part of the
+  variables for "system tunning". These shouldn't be changed.
+
+
+.. _vagrant_file:
+
+``Vagrantfile``
+---------------
+
+The ``Vagrantfile`` should not need to change, and it is symlinked back to the
+``Vagrantfile`` that exists in the root of the project. It is linked in this
+way so that a vagrant environment can be isolated to the given scenario.
+
+
+.. _hosts_file:
+
+``hosts``
+---------
+
+The ``hosts`` file should contain the hosts needed for the scenario. This might
+seem a bit repetitive since machines are already defined in
+:ref:`vagrant_variables` but it allows granular changes to hosts (for example
+defining different public_network values between monitors) which can help catch issues in
+``ceph-ansible`` configuration. For example:
+
+.. code-block:: ini
+
+   [mons]
+   mon0 public_network=192.168.1.0/24
+   mon1 public_network=192.168.2.0/24
+   mon2 public_network=192.168.3.0/24
+
+.. _group_vars:
+
+``group_vars``
+--------------
+
+This directory holds any configuration change that will affect ``ceph-ansible``
+deployments in the same way as if ansible was executed from the root of the
+project.
+
+The file that will need to be defined always is ``all`` where (again) certain
+values like ``public_network`` and ``cluster_network`` will need to be defined
+along with any customizations that ``ceph-ansible`` supports.
+
+
+.. _scenario_wiring:
+
+Scenario Wiring
+---------------
+
+Scenarios are just meant to provide the Ceph environment for testing, but they
+do need to be defined in the ``tox.ini`` so that they are available to the test
+framework. To see a list of available scenarios, the following command (ran
+from the root of the project) will list them, shortened for brevity:
+
+.. code-block:: console
+
+   $ tox -l
+   ...
+   luminous-ansible2.4-centos7_cluster
+   ...
+
+These scenarios are made from different variables, in the above command there
+are 3:
+
+* ``jewel``: the Ceph version to test
+* ``ansible2.4``: the Ansible version to install
+* ``centos7_cluster``: the name of the scenario
+
+The last one is important in the *wiring up* of the scenario. It is a variable
+that will define in what path the scenario lives. For example, the
+``changedir`` section for ``centos7_cluster`` that looks like:
+
+.. code-block:: ini
+
+   centos7_cluster: {toxinidir}/tests/functional/centos/7/cluster
+
+The actual tests are written for specific daemon types, for all daemon types,
+and for specific use cases (e.g. journal collocation), those have their own
+conventions as well which are explained in detail in :ref:`test_conventions`
+and :ref:`test_files`.
+
+As long as a test scenario defines OSDs and MONs, the OSD tests and MON tests
+will run.
+
+
+.. _scenario_conventions:
+
+Conventions
+-----------
+
+.. _scenario_environment_configuration:
+
+Environment configuration
+-------------------------
+
+.. _scenario_ansible_configuration:
+
+Ansible configuration
+---------------------

docs/source/testing/tests.rst

@@ -0,0 +1,99 @@
+.. _tests:
+
+Tests
+=====
+
+Actual tests are written in Python methods that accept optional fixtures. These
+fixtures come with interesting attributes to help with remote assertions.
+
+As described in :ref:`test_conventions`, tests need to go into
+``tests/functional/tests/``. These are collected and *mapped* to a distinct
+node type, or *mapped* to run on all nodes.
+
+Simple Python asserts are used (these tests do not need to follow the Python
+``unittest.TestCase`` base class) that make it easier to reason about failures
+and errors.
+
+The test run is handled by ``py.test`` along with :ref:`testinfra` for handling
+remote execution.
+
+
+.. _test_files:
+
+Test Files
+----------
+
+
+
+.. _test_fixtures:
+
+Test Fixtures
+=============
+
+Test fixtures are a powerful feature of ``py.test`` and most tests depend on
+this for making assertions about remote nodes. To request them in a test
+method, all that is needed is to require it as an argument.
+
+Fixtures are detected by name, so as long as the argument being used has the
+same name, the fixture will be passed in (see `pytest fixtures`_ for more
+in-depth examples). The code that follows shows a test method that will use the
+``node`` fixture that contains useful information about a node in a ceph
+cluster:
+
+.. code-block:: python
+
+   def test_ceph_conf(self, node):
+       assert node['conf_path'] == "/etc/ceph/ceph.conf"
+
+The test is naive (the configuration path might not exist remotely) but
+explains how simple it is to "request" a fixture.
+
+For remote execution, we can rely further on other fixtures (tests can have as
+many fixtures as needed) like ``File``:
+
+.. code-block:: python
+
+   def test_ceph_config_has_inital_members_line(self, node, File):
+       assert File(node["conf_path"]).contains("^mon initial members = .*$")
+
+
+.. _node:
+
+``node`` fixture
+----------------
+
+The ``node`` fixture contains a few useful pieces of information about the node
+where the test is being executed, this is captured once, before tests run:
+
+* ``address``: The IP for the ``eth1`` interface
+* ``subnet``: The subnet that ``address`` belongs to
+* ``vars``: all the Ansible vars set for the current run
+* ``osd_ids``: a list of all the OSD IDs
+* ``num_mons``: the total number of monitors for the current environment
+* ``num_devices``: the number of devices for the current node
+* ``num_osd_hosts``: the total number of OSD hosts
+* ``total_osds``: total number of OSDs on the current node
+* ``cluster_name``: the name of the Ceph cluster (which defaults to 'ceph')
+* ``conf_path``: since the cluster name can change the file path for the Ceph
+  configuration, this gets sets according to the cluster name.
+* ``cluster_address``: the address used for cluster communication. All
+  environments are set up with 2 interfaces, 1 being used exclusively for the
+  cluster
+* ``docker``: A boolean that identifies a Ceph Docker cluster
+* ``osds``: A list of OSD IDs, unless it is a Docker cluster, where it gets the
+  name of the devices (e.g. ``sda1``)
+
+
+Other Fixtures
+--------------
+
+There are a lot of other fixtures provided by :ref:`testinfra` as well as
+``py.test``. The full list of ``testinfra`` fixtures are available in
+`testinfra_fixtures`_
+
+``py.test`` builtin fixtures can be listed with ``pytest -q --fixtures`` and
+they are described in `pytest builtin fixtures`_
+
+.. _pytest fixtures: https://docs.pytest.org/en/latest/fixture.html
+.. _pytest builtin fixtures: https://docs.pytest.org/en/latest/builtin.html#builtin-fixtures-function-arguments
+.. _testinfra_fixtures: https://testinfra.readthedocs.io/en/latest/modules.html#modules

docs/source/testing/tox.rst

@@ -0,0 +1,75 @@
+.. _tox:
+
+``tox``
+=======
+
+``tox`` is an automation project we use to run our testing scenarios. It gives us
+the ability to create a dynamic matrix of many testing scenarios, isolated testing environments
+and a provides a single entry point to run all tests in an automated and repeatable fashion.
+
+Documentation for tox can be found `here <https://tox.readthedocs.io/en/latest/>`_.
+
+
+.. _tox_environment_variables:
+
+Environment variables
+---------------------
+
+When running ``tox`` we've allowed for the usage of environment variables to tweak certain settings
+of the playbook run using Ansible's ``--extra-vars``. It's helpful in Jenkins jobs or for manual test
+runs of ``ceph-ansible``.
+
+The following environent variables are available for use:
+
+* ``CEPH_DOCKER_REGISTRY``: (default: ``quay.io``) This would configure the ``ceph-ansible`` variable ``ceph_docker_registry``.
+
+* ``CEPH_DOCKER_IMAGE``: (default: ``ceph/daemon``) This would configure the ``ceph-ansible`` variable ``ceph_docker_image``.
+
+* ``CEPH_DOCKER_IMAGE_TAG``: (default: ``latest``) This would configure the ``ceph-ansible`` variable ``ceph_docker_image_name``.
+
+* ``CEPH_DEV_BRANCH``: (default: ``main``) This would configure the ``ceph-ansible`` variable ``ceph_dev_branch`` which defines which branch we'd
+  like to install from shaman.ceph.com.
+
+* ``CEPH_DEV_SHA1``: (default: ``latest``) This would configure the ``ceph-ansible`` variable ``ceph_dev_sha1`` which defines which sha1 we'd like
+  to install from shaman.ceph.com.
+
+* ``UPDATE_CEPH_DEV_BRANCH``: (default: ``main``) This would configure the ``ceph-ansible`` variable ``ceph_dev_branch`` which defines which branch we'd
+  like to update to from shaman.ceph.com.
+
+* ``UPDATE_CEPH_DEV_SHA1``: (default: ``latest``) This would configure the ``ceph-ansible`` variable ``ceph_dev_sha1`` which defines which sha1 we'd like
+  to update to from shaman.ceph.com.
+
+
+.. _tox_sections:
+
+Sections
+--------
+
+The ``tox.ini`` file has a number of top level sections defined by ``[ ]`` and subsections within those. For complete documentation
+on all subsections inside of a tox section please refer to the tox documentation.
+
+* ``tox`` : This section contains the ``envlist`` which is used to create our dynamic matrix. Refer to the `section here <http://tox.readthedocs.io/en/latest/config.html#generating-environments-conditional-settings>`_ for more information on how the ``envlist`` works. 
+
+* ``purge`` : This section contains commands that only run for scenarios that purge the cluster and redeploy. You'll see this section being reused in ``testenv``
+  with the following syntax: ``{[purge]commands}``
+
+* ``update`` : This section contains commands taht only run for scenarios that deploy a cluster and then upgrade it to another Ceph version.
+
+* ``testenv`` : This is the main section of the ``tox.ini`` file and is run on every scenario. This section contains many *factors* that define conditional
+  settings depending on the scenarios defined in the ``envlist``. For example, the factor ``centos7_cluster`` in the ``changedir`` subsection of ``testenv`` sets
+  the directory that tox will change do when that factor is selected. This is an important behavior that allows us to use the same ``tox.ini`` and reuse commands while
+  tweaking certain sections per testing scenario.
+
+
+.. _tox_environments:
+
+Modifying or Adding environments
+--------------------------------
+
+The tox environments are controlled by the ``envlist`` subsection of the ``[tox]`` section. Anything inside of ``{}`` is considered a *factor* and will be included
+in the dynamic matrix that tox creates. Inside of ``{}`` you can include a comma separated list of the *factors*. Do not use a hyphen (``-``) as part
+of the *factor* name as those are used by tox as the separator between different factor sets.
+
+For example, if wanted to add a new test *factor* for the next Ceph release of luminious this is how you'd accomplish that. Currently, the first factor set in our ``envlist``
+is used to define the Ceph release (``{jewel,kraken}-...``). To add luminous you'd change that to look like ``{luminous,kraken}-...``. In the ``testenv`` section
+this is a subsection called ``setenv`` which allows you to provide environment variables to the tox environment and we support an environment variable called ``CEPH_STABLE_RELEASE``. To ensure that all the new tests that are created by adding the luminous *factor* you'd do this in that section: ``luminous: CEPH_STABLE_RELEASE=luminous``.

docs/tox.ini

@@ -0,0 +1,10 @@
+[tox]
+envlist = docs
+skipsdist = True
+
+[testenv:docs]
+basepython=python
+changedir=source
+deps=sphinx==1.7.9
+commands=
+    sphinx-build -W -b html -d {envtmpdir}/doctrees .  {envtmpdir}/html