From 8c489009c238aa6d56354f5ef873addea1f4fd5c Mon Sep 17 00:00:00 2001
From: Jason Anderson <jasonanderson@uchicago.edu>
Date: Mon, 24 Feb 2020 15:36:46 -0600
Subject: [PATCH] [elasticsearch] Add migration for Kibana 6.x index
Elasticsearch 6.x dropped support for mapping types[1], which by default
the Kibana index used. This means that when deploying ELK 6.x, the
Kibana index must be migrated to the new schema to preserve dashboards
and visualizations. There is a process defined[2], which involves
creating a new index with the specified schema, then reindexing the old
index's data into the new index, then doing a rename/delete.
This adds support for that workflow via Ansible. It takes place after
the ES container is restarted after an upgrade, so there will be a
(short) period of time where the Kibana index is not migrated. During
this time, Kibana still loads, but presents the user with a status
screen informing that the index needs migration.
[1]:
https://www.elastic.co/guide/en/elasticsearch/reference/6.x/removal-of-types.html
[2]: https://www.elastic.co/guide/en/kibana/6.x/migrating-6.0-index.html
Implements: blueprint elasticsearch-kibana-version-upgrade
Depends-On: https://review.opendev.org/709624
Change-Id: I4550629e2113f3da7f1cecfeab0d5fe0d899dae8
---
.../roles/kibana/files/kibana-6-index.json | 264 ++++++++++++++++++
.../kibana/tasks/migrate-kibana-index.yml | 111 ++++++++
ansible/roles/kibana/tasks/upgrade.yml | 2 +
3 files changed, 377 insertions(+)
create mode 100644 ansible/roles/kibana/files/kibana-6-index.json
create mode 100644 ansible/roles/kibana/tasks/migrate-kibana-index.yml
diff --git a/ansible/roles/kibana/files/kibana-6-index.json b/ansible/roles/kibana/files/kibana-6-index.json
new file mode 100644
index 0000000000..08e61bb0d1
--- /dev/null
+++ b/ansible/roles/kibana/files/kibana-6-index.json
@@ -0,0 +1,264 @@
+{
+ "settings" : {
+ "number_of_shards" : 1,
+ "index.mapper.dynamic": false
+ },
+ "mappings" : {
+ "doc": {
+ "properties": {
+ "type": {
+ "type": "keyword"
+ },
+ "updated_at": {
+ "type": "date"
+ },
+ "config": {
+ "properties": {
+ "buildNum": {
+ "type": "keyword"
+ }
+ }
+ },
+ "index-pattern": {
+ "properties": {
+ "fieldFormatMap": {
+ "type": "text"
+ },
+ "fields": {
+ "type": "text"
+ },
+ "intervalName": {
+ "type": "keyword"
+ },
+ "notExpandable": {
+ "type": "boolean"
+ },
+ "sourceFilters": {
+ "type": "text"
+ },
+ "timeFieldName": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ }
+ }
+ },
+ "visualization": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "savedSearchId": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ },
+ "uiStateJSON": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ },
+ "visState": {
+ "type": "text"
+ }
+ }
+ },
+ "search": {
+ "properties": {
+ "columns": {
+ "type": "keyword"
+ },
+ "description": {
+ "type": "text"
+ },
+ "hits": {
+ "type": "integer"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "sort": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "dashboard": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "hits": {
+ "type": "integer"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "optionsJSON": {
+ "type": "text"
+ },
+ "panelsJSON": {
+ "type": "text"
+ },
+ "refreshInterval": {
+ "properties": {
+ "display": {
+ "type": "keyword"
+ },
+ "pause": {
+ "type": "boolean"
+ },
+ "section": {
+ "type": "integer"
+ },
+ "value": {
+ "type": "integer"
+ }
+ }
+ },
+ "timeFrom": {
+ "type": "keyword"
+ },
+ "timeRestore": {
+ "type": "boolean"
+ },
+ "timeTo": {
+ "type": "keyword"
+ },
+ "title": {
+ "type": "text"
+ },
+ "uiStateJSON": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "url": {
+ "properties": {
+ "accessCount": {
+ "type": "long"
+ },
+ "accessDate": {
+ "type": "date"
+ },
+ "createDate": {
+ "type": "date"
+ },
+ "url": {
+ "type": "text",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 2048
+ }
+ }
+ }
+ }
+ },
+ "server": {
+ "properties": {
+ "uuid": {
+ "type": "keyword"
+ }
+ }
+ },
+ "timelion-sheet": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "hits": {
+ "type": "integer"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "timelion_chart_height": {
+ "type": "integer"
+ },
+ "timelion_columns": {
+ "type": "integer"
+ },
+ "timelion_interval": {
+ "type": "keyword"
+ },
+ "timelion_other_interval": {
+ "type": "keyword"
+ },
+ "timelion_rows": {
+ "type": "integer"
+ },
+ "timelion_sheet": {
+ "type": "text"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "graph-workspace": {
+ "properties": {
+ "description": {
+ "type": "text"
+ },
+ "kibanaSavedObjectMeta": {
+ "properties": {
+ "searchSourceJSON": {
+ "type": "text"
+ }
+ }
+ },
+ "numLinks": {
+ "type": "integer"
+ },
+ "numVertices": {
+ "type": "integer"
+ },
+ "title": {
+ "type": "text"
+ },
+ "version": {
+ "type": "integer"
+ },
+ "wsState": {
+ "type": "text"
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/ansible/roles/kibana/tasks/migrate-kibana-index.yml b/ansible/roles/kibana/tasks/migrate-kibana-index.yml
new file mode 100644
index 0000000000..694a5a8d96
--- /dev/null
+++ b/ansible/roles/kibana/tasks/migrate-kibana-index.yml
@@ -0,0 +1,111 @@
+---
+- name: Set fact for Elasticsearch URL
+ set_fact:
+ elasticsearch_url: "{{ internal_protocol }}://{{ kolla_internal_vip_address | put_address_in_context('url') }}:{{ elasticsearch_port }}"
+
+- name: Wait for Elasticsearch
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}"
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+ retries: 10
+ delay: 5
+ register: result
+ until: ('status' in result) and result.status == 200
+
+- name: Check state of migration
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}/.kibana/_mappings/doc"
+ status_code: [200, 404]
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+ register: kibana_6_index
+
+# The official procedure for migrating the Kibana index:
+# https://www.elastic.co/guide/en/kibana/6.x/migrating-6.0-index.html
+- name: Migrate Kibana index to 6.x
+ block:
+ - name: Set .kibana index to read-only
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}/.kibana/_settings"
+ method: PUT
+ status_code: 200
+ return_content: yes
+ body: |
+ {
+ "index.blocks.write": true
+ }
+ body_format: json
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+
+ - name: Create .kibana-6 index
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}/.kibana-6"
+ method: PUT
+ status_code: 200
+ return_content: yes
+ body: "{{ lookup('file', 'kibana-6-index.json') }}"
+ body_format: json
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+
+ - name: Reindex .kibana into .kibana-6
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}/_reindex"
+ method: POST
+ status_code: 200
+ return_content: yes
+ body: |
+ {
+ "source": {
+ "index": ".kibana"
+ },
+ "dest": {
+ "index": ".kibana-6"
+ },
+ "script": {
+ "inline": "ctx._source = [ ctx._type : ctx._source ]; ctx._source.type = ctx._type; ctx._id = ctx._type + \":\" + ctx._id; ctx._type = \"doc\"; ",
+ "lang": "painless"
+ }
+ }
+ body_format: json
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+
+ - name: Alias .kibana-6 to .kibana and remove legacy .kibana index
+ become: true
+ kolla_toolbox:
+ module_name: uri
+ module_args:
+ url: "{{ elasticsearch_url }}/_aliases"
+ method: POST
+ status_code: 200
+ return_content: yes
+ body: |
+ {
+ "actions" : [
+ { "add": { "index": ".kibana-6", "alias": ".kibana" } },
+ { "remove_index": { "index": ".kibana" } }
+ ]
+ }
+ body_format: json
+ delegate_to: "{{ groups['elasticsearch'][0] }}"
+ run_once: true
+
+ when: ('status' in kibana_6_index) and kibana_6_index.status != 200
diff --git a/ansible/roles/kibana/tasks/upgrade.yml b/ansible/roles/kibana/tasks/upgrade.yml
index 375dcad19b..4192da0fe5 100644
--- a/ansible/roles/kibana/tasks/upgrade.yml
+++ b/ansible/roles/kibana/tasks/upgrade.yml
@@ -1,5 +1,7 @@
---
- include_tasks: config.yml
+- include_tasks: migrate-kibana-index.yml
+
- name: Flush handlers
meta: flush_handlers
--
GitLab