diff --git a/ansible/roles/ceph/tasks/bootstrap_osds.yml b/ansible/roles/ceph/tasks/bootstrap_osds.yml
index 16599170e6f1f3905cb44a35fd0f2f09f6eee2db..471d173c90194cc9a2f87e78fd71a6fe23bd5529 100644
--- a/ansible/roles/ceph/tasks/bootstrap_osds.yml
+++ b/ansible/roles/ceph/tasks/bootstrap_osds.yml
@@ -2,7 +2,7 @@
- name: Looking up disks to bootstrap for Ceph
command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
-m find_disks
- -a "partition_name='KOLLA_CEPH_OSD_BOOTSTRAP'"
+ -a "partition_name='KOLLA_CEPH_OSD_BOOTSTRAP' match_mode='prefix'"
register: osd_lookup
changed_when: "{{ osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -14,7 +14,7 @@
- name: Looking up disks to bootstrap for Ceph
command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
-m find_disks
- -a "partition_name='KOLLA_CEPH_OSD_CACHE_BOOTSTRAP'"
+ -a "partition_name='KOLLA_CEPH_OSD_CACHE_BOOTSTRAP' match_mode='prefix'"
register: osd_cache_lookup
changed_when: "{{ osd_cache_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_cache_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
failed_when: osd_cache_lookup.stdout.split()[2] != 'SUCCESS'
@@ -32,6 +32,12 @@
KOLLA_BOOTSTRAP:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
OSD_DEV: "{{ item.1.device }}"
+ OSD_PARTITION: "{{ item.1.partition }}"
+ OSD_PARTITION_NUM: "{{ item.1.partition_num }}"
+ JOURNAL_DEV: "{{ item.1.journal_device }}"
+ JOURNAL_PARTITION: "{{ item.1.journal }}"
+ JOURNAL_PARTITION_NUM: "{{ item.1.journal_num }}"
+ USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}"
OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
@@ -56,6 +62,12 @@
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
CEPH_CACHE:
OSD_DEV: "{{ item.1.device }}"
+ OSD_PARTITION: "{{ item.1.partition }}"
+ OSD_PARTITION_NUM: "{{ item.1.partition_num }}"
+ JOURNAL_DEV: "{{ item.1.journal_device }}"
+ JOURNAL_PARTITION: "{{ item.1.journal }}"
+ JOURNAL_PARTITION_NUM: "{{ item.1.journal_num }}"
+ USE_EXTERNAL_JOURNAL: "{{ item.1.external_journal | bool }}"
OSD_FILESYSTEM: "{{ ceph_osd_filesystem }}"
OSD_INITIAL_WEIGHT: "{{ osd_initial_weight }}"
HOSTNAME: "{{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}"
diff --git a/ansible/roles/ceph/tasks/start_osds.yml b/ansible/roles/ceph/tasks/start_osds.yml
index 3f2bdd4cdc3695d8ec287d5b8cf8918b938cec71..63787dd120340c6f6809ec274ddaeb6b3ebf707e 100644
--- a/ansible/roles/ceph/tasks/start_osds.yml
+++ b/ansible/roles/ceph/tasks/start_osds.yml
@@ -2,7 +2,7 @@
- name: Looking up OSDs for Ceph
command: docker exec -t kolla_toolbox /usr/bin/ansible localhost
-m find_disks
- -a "partition_name='KOLLA_CEPH_DATA'"
+ -a "partition_name='KOLLA_CEPH_DATA' match_mode='prefix'"
register: osd_lookup
changed_when: "{{ osd_lookup.stdout.find('localhost | SUCCESS => ') != -1 and (osd_lookup.stdout.split('localhost | SUCCESS => ')[1]|from_json).changed }}"
failed_when: osd_lookup.stdout.split()[2] != 'SUCCESS'
@@ -34,7 +34,7 @@
environment:
KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
OSD_ID: "{{ item.0.stdout }}"
- OSD_DEV: "{{ item.1['device'] }}"
+ JOURNAL_PARTITION: "{{ item.1.journal }}"
image: "{{ ceph_osd_image_full }}"
name: "ceph_osd_{{ item.0.stdout }}"
pid_mode: "host"
diff --git a/docker/ceph/ceph-osd/extend_start.sh b/docker/ceph/ceph-osd/extend_start.sh
index 5db6f61c9358fdf4159f7f75560f0b024790058e..97119578a753704c666b7e84079bd313331d2fe8 100644
--- a/docker/ceph/ceph-osd/extend_start.sh
+++ b/docker/ceph/ceph-osd/extend_start.sh
@@ -3,28 +3,22 @@
# Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
# of the KOLLA_BOOTSTRAP variable being set, including empty.
if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
+ # NOTE(SamYaple): Static gpt partcodes
+ CEPH_JOURNAL_TYPE_CODE="45B0969E-9B03-4F30-B4C6-B4B80CEFF106"
+ CEPH_OSD_TYPE_CODE="4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D"
+
# Wait for ceph quorum before proceeding
ceph quorum_status
- # Formatting disk for ceph
- sgdisk --zap-all -- "${OSD_DEV}"
- sgdisk --new=2:1M:5G --change-name=2:KOLLA_CEPH_JOURNAL --typecode=2:45B0969E-9B03-4F30-B4C6-B4B80CEFF106 --mbrtogpt -- "${OSD_DEV}"
- sgdisk --largest-new=1 --change-name=1:KOLLA_CEPH_DATA --typecode=1:4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D -- "${OSD_DEV}"
- # This command may throw errors that we can safely ignore
- partprobe || true
+ if [[ "${USE_EXTERNAL_JOURNAL}" == "False" ]]; then
+ # Formatting disk for ceph
+ sgdisk --zap-all -- "${OSD_DEV}"
+ sgdisk --new=2:1M:5G -- "${JOURNAL_DEV}"
+ sgdisk --largest-new=1 -- "${OSD_DEV}"
+ # NOTE(SamYaple): This command may throw errors that we can safely ignore
+ partprobe || true
- count=0
- while [[ "${OSD_PARTITION}x" == "x" ]]; do
- if [[ "${count}" -gt 5 ]]; then
- echo "Could not find OSD Partition"
- exit 1
- fi
- sleep 1
- # We look up the appropriate device path with partition.
- OSD_PARTITION=$(ls "${OSD_DEV}"* | egrep "${OSD_DEV}p?1")
- count=$(( count + 1 ))
- done
- JOURNAL_PARTITION="${OSD_PARTITION%?}2"
+ fi
OSD_ID=$(ceph osd create)
OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
@@ -56,10 +50,13 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
# Adding osd to crush map
ceph osd crush add "${OSD_ID}" "${OSD_INITIAL_WEIGHT}" host="${HOSTNAME}${CEPH_ROOT_NAME:+-${CEPH_ROOT_NAME}}"
+
+ # Setting partition name based on ${OSD_ID}
+ sgdisk "--change-name=${OSD_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}" "--typecode=${OSD_PARTITION_NUM}:${CEPH_OSD_TYPE_CODE}" -- "${OSD_DEV}"
+ sgdisk "--change-name=${JOURNAL_PARTITION_NUM}:KOLLA_CEPH_DATA_${OSD_ID}_J" "--typecode=${JOURNAL_PARTITION_NUM}:${CEPH_JOURNAL_TYPE_CODE}" -- "${JOURNAL_DEV}"
+
exit 0
fi
-# We look up the appropriate journal since we cannot rely on symlinks
-JOURNAL_PARTITION=$(ls "${OSD_DEV}"* | egrep "${OSD_DEV}p?2")
OSD_DIR="/var/lib/ceph/osd/ceph-${OSD_ID}"
ARGS="-i ${OSD_ID} --osd-journal ${JOURNAL_PARTITION} -k ${OSD_DIR}/keyring"
diff --git a/docker/kolla-toolbox/find_disks.py b/docker/kolla-toolbox/find_disks.py
index 6ebf2b6a275e76bd23e3916e12f9eb9f250c3077..e72dcc6fba6e311434d5b14b58c16a1eb31785ce 100644
--- a/docker/kolla-toolbox/find_disks.py
+++ b/docker/kolla-toolbox/find_disks.py
@@ -66,6 +66,61 @@ EXAMPLES = '''
import json
import pyudev
+import re
+
+
+def is_dev_matched_by_name(dev, name, mode):
+ if dev.get('DEVTYPE', '') == 'partition':
+ dev_name = dev.get('ID_PART_ENTRY_NAME', '')
+ else:
+ dev_name = dev.get('ID_FS_LABEL', '')
+
+ if mode == 'strict':
+ return dev_name == name
+ elif mode == 'prefix':
+ return dev_name.startswith(name)
+ else:
+ return False
+
+
+def find_disk(ct, name, match_mode):
+ for dev in ct.list_devices(subsystem='block'):
+ if is_dev_matched_by_name(dev, name, match_mode):
+ yield dev
+
+
+def extract_disk_info(ct, dev, name):
+ if not dev:
+ return
+ kwargs = dict()
+ kwargs['fs_uuid'] = dev.get('ID_FS_UUID', '')
+ kwargs['fs_label'] = dev.get('ID_FS_LABEL', '')
+ if dev.get('DEVTYPE', '') == 'partition':
+ kwargs['device'] = dev.find_parent('block').device_node
+ kwargs['partition'] = dev.device_node
+ kwargs['partition_num'] = \
+ re.sub(r'.*[^\d$]', '', dev.device_node)
+ if is_dev_matched_by_name(dev, name, 'strict'):
+ kwargs['external_journal'] = False
+ kwargs['journal'] = dev.device_node[:-1] + '2'
+ kwargs['journal_device'] = kwargs['device']
+ kwargs['journal_num'] = 2
+ else:
+ kwargs['external_journal'] = True
+ journal_name = dev.get('ID_PART_ENTRY_NAME', '') + '_J'
+ for journal in find_disk(ct, journal_name, 'strict'):
+ kwargs['journal'] = journal.device_node
+ kwargs['journal_device'] = \
+ journal.find_parent('block').device_node
+ kwargs['journal_num'] = \
+ re.sub(r'.*[^\d$]', '', journal.device_node)
+ break
+ if 'journal' not in kwargs:
+ # NOTE(SamYaple): Journal not found, not returning info
+ return
+ else:
+ kwargs['device'] = dev.device_node
+ yield kwargs
def main():
@@ -78,33 +133,14 @@ def main():
match_mode = module.params.get('match_mode')
name = module.params.get('name')
- def is_dev_matched_by_name(dev, name):
- if dev.get('DEVTYPE', '') == 'partition':
- dev_name = dev.get('ID_PART_ENTRY_NAME', '')
- else:
- dev_name = dev.get('ID_FS_LABEL', '')
-
- if match_mode == 'strict':
- return dev_name == name
- elif match_mode == 'prefix':
- return dev_name.startswith(name)
- else:
- return False
-
try:
ret = list()
ct = pyudev.Context()
- for dev in ct.list_devices(subsystem='block'):
- if is_dev_matched_by_name(dev, name):
- fs_uuid = dev.get('ID_FS_UUID', '')
- fs_label = dev.get('ID_FS_LABEL', '')
- if dev.get('DEVTYPE', '') == 'partition':
- device_node = dev.find_parent('block').device_node
- else:
- device_node = dev.device_node
- ret.append({'device': device_node,
- 'fs_uuid': fs_uuid,
- 'fs_label': fs_label})
+ for dev in find_disk(ct, name, match_mode):
+ for info in extract_disk_info(ct, dev, name):
+ if info:
+ ret.append(info)
+
module.exit_json(disks=json.dumps(ret))
except Exception as e:
module.exit_json(failed=True, msg=repr(e))