From 2392d7e38cc5937921c2ecaf5af2b21f3cb32c40 Mon Sep 17 00:00:00 2001
From: Jeffrey Zhang <zhang.lei.fly@gmail.com>
Date: Thu, 20 Oct 2016 02:21:35 +0800
Subject: [PATCH] Add file validator for *.json.j2 file
All json file in Kolla are json.j2 file, it shold be render as jinja
template, then validate by using json.
Change-Id: Ibabbe435116fd255d68347e00407156db76fedfe
Closes-Bug: #1634447
---
tools/validate-all-file.py | 57 ++++++++++++++++++++++++++++++++++++--
tools/validate-all-json.sh | 7 -----
tools/validate-json.py | 44 -----------------------------
tox.ini | 4 +--
4 files changed, 56 insertions(+), 56 deletions(-)
delete mode 100755 tools/validate-all-json.sh
delete mode 100755 tools/validate-json.py
diff --git a/tools/validate-all-file.py b/tools/validate-all-file.py
index 601f2e432..e44641291 100755
--- a/tools/validate-all-file.py
+++ b/tools/validate-all-file.py
@@ -12,18 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import collections
import fnmatch
+import json
import logging
import os
import re
import sys
+import jinja2
+
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
NEWLINE_EOF_INCLUDE_PATTERNS = ['*.j2', '*.yml', '*.py', '*.sh']
NEWLINE_EOF_EXCLUDE_PATTERNS = ['.tox', '.testrepository', '.git']
+# Render json file by using jinja2 template is OK
+JSON_J2_INCLUDE_PATTERNS = ['*.json.j2', '*.json']
+JSON_J2_EXCLUDE_PATTERNS = ['.tox', '.testrepository', '.git']
+
logging.basicConfig()
LOG = logging.getLogger(__name__)
@@ -52,9 +60,54 @@ def check_newline_eof():
return return_code
-def main():
- return sum([check_newline_eof()])
+def check_json_j2():
+ includes = r'|'.join([fnmatch.translate(x)
+ for x in JSON_J2_INCLUDE_PATTERNS])
+ excludes = r'|'.join([fnmatch.translate(x)
+ for x in JSON_J2_EXCLUDE_PATTERNS])
+ return_code = 0
+
+ def bool_filter(value):
+ return True
+ # Mock ansible hostvars variable, which is a nested dict
+ def hostvars():
+ return collections.defaultdict(hostvars)
+
+ def validate_json_j2(root, filename):
+ env = jinja2.Environment( # nosec: not used to render HTML
+ loader=jinja2.FileSystemLoader(root))
+ env.filters['bool'] = bool_filter
+ template = env.get_template(filename)
+ # Mock ansible variables.
+ context = {
+ 'hostvars': hostvars(),
+ 'cluster_interface': 'cluster_interface',
+ 'storage_interface': 'storage_interface',
+ 'inventory_hostname': 'hostname'
+ }
+ data = template.render(**context)
+ json.loads(data)
+ for root, dirs, files in os.walk(PROJECT_ROOT):
+ dirs[:] = [d for d in dirs if not re.match(excludes, d)]
+ for filename in files:
+ if not re.match(excludes, filename) and \
+ re.match(includes, filename):
+ fullpath = os.path.join(root, filename)
+ try:
+ validate_json_j2(root, filename)
+ except (ValueError, jinja2.exceptions.TemplateError):
+ return_code = 1
+ LOG.exception('%s file error', fullpath)
+ return return_code
+
+
+def main():
+ checks = (
+ check_newline_eof,
+ check_json_j2
+ )
+ return sum([check() for check in checks])
if __name__ == "__main__":
sys.exit(main())
diff --git a/tools/validate-all-json.sh b/tools/validate-all-json.sh
deleted file mode 100755
index 7e61a8a1a..000000000
--- a/tools/validate-all-json.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-REAL_PATH=$(python -c "import os,sys;print os.path.realpath('$0')")
-cd "$(dirname "$REAL_PATH")/.."
-
-find . -path ./.tox -prune -name '*.json' -print0 |
- xargs -0 python tools/validate-json.py || exit 1
diff --git a/tools/validate-json.py b/tools/validate-json.py
deleted file mode 100755
index eee2276db..000000000
--- a/tools/validate-json.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/python
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import argparse
-import json
-import logging
-import sys
-
-
-def parse_args():
- p = argparse.ArgumentParser()
- p.add_argument('input', nargs='*')
- return p.parse_args()
-
-
-def main():
- args = parse_args()
- logging.basicConfig()
- res = 0
-
- for filename in args.input:
- with open(filename) as fd:
- try:
- json.load(fd)
- except ValueError as error:
- res = 1
- logging.error('%s failed validation: %s',
- filename, error)
-
- sys.exit(res)
-
-if __name__ == '__main__':
- main()
diff --git a/tox.ini b/tox.ini
index ece81a646..5bbdcc0c8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,10 +25,8 @@ commands = oslo_debug_helper {posargs}
commands =
{toxinidir}/tools/run-bashate.sh
flake8 {posargs}
- {toxinidir}/tools/validate-all-json.sh
- {toxinidir}/tools/validate-all-yaml.sh
{toxinidir}/tools/validate-all-dockerfiles.sh
- {toxinidir}/tools/validate-all-file.py
+ python {toxinidir}/tools/validate-all-file.py
bandit -r ansible/library docker kolla tests tools
[testenv:bandit]
--
GitLab