compare-results.py

#!/usr/bin/env python3
# Copyright 2018 sysmocom - s.f.m.c. GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import re

doc = "Compare TTCN3 test run results with expected results by junit logs."

# The nicest would be to use an XML library, but I don't want to introduce dependencies on the build slaves.
re_testcase = re.compile(r'''<testcase classname=['"]([^'"]+)['"].* name=['"]([^'"]+)['"].*>''')
re_testcase_end = re.compile(r'''(</testcase>|<testcase [^>]*/>)''')
re_failure = re.compile(r'''(<failure\b|<error\b)''')

RESULT_PASS = 'pass'
RESULT_FAIL = 'pass->FAIL'
RESULT_SKIP = 'skip'
RESULT_XFAIL = 'xfail'
RESULT_FIXED = 'xfail->PASS'
RESULT_NEW_PASS = 'NEW: PASS'
RESULT_NEW_FAIL = 'NEW: FAIL'

RESULTS = (
	RESULT_FAIL,
	RESULT_NEW_FAIL,
	RESULT_XFAIL,
	RESULT_FIXED,
	RESULT_PASS,
	RESULT_NEW_PASS,
	RESULT_SKIP,
	)

def count(counter, name, result):
	v = counter.get(result) or 0
	v += 1
	counter[result] = v
	if result != RESULT_SKIP:
		print('%s %s' % (result, name))

def compare_one(name, expect, result, counter):
	if result is None:
		count(counter, name, RESULT_SKIP)
	elif result == RESULT_PASS:
		if expect == RESULT_PASS:
			count(counter, name, RESULT_PASS)
		elif expect == RESULT_FAIL:
			count(counter, name, RESULT_FIXED)
		elif expect is None:
			count(counter, name, RESULT_NEW_PASS)
	elif result == RESULT_FAIL:
		if expect == RESULT_PASS:
			count(counter, name, RESULT_FAIL)
		elif expect == RESULT_FAIL:
			count(counter, name, RESULT_XFAIL)
		elif expect is None:
			count(counter, name, RESULT_NEW_FAIL)

def compare(cmdline, f_expected, f_current):
	expected_list = parse_results(f_expected)
	current_list = parse_results(f_current)

	expected_dict = dict(expected_list)
	current_dict = dict(current_list)

	counter = {}

	for expected_name, expected_result in expected_list:
		compare_one(expected_name, expected_result, current_dict.get(expected_name), counter)

	# Also count new tests
	for current_name, current_result in current_list:
		if current_name in expected_dict:
			continue
		compare_one(current_name, None, current_result, counter)


	print('\nSummary:')
	for r in RESULTS:
		v = counter.get(r)
		if not v:
			continue
		print('  %s: %d' % (r, v))
	print('\n')

def parse_results(f):
	tests = []
	name = None
	result = None
	for line in f:
		m = re_testcase.search(line)
		if m:
			class_name, test_name = m.groups()
			name = '%s.%s' % (class_name, test_name)

		m = re_failure.search(line)
		if m:
			result = RESULT_FAIL

		m = re_testcase_end.search(line)
		if m:
			if not name:
				continue
			if result is None:
				result = RESULT_PASS
			tests.append((name, result))

			name = None
			result = None

	return tests

def main(cmdline):
	with open(cmdline.expected_results, 'r') as f_expected:
		with open(cmdline.current_results, 'r') as f_current:
			print('\nComparing expected results %r against results in %r\n--------------------'
				% (cmdline.expected_results, cmdline.current_results))
			compare(cmdline, f_expected, f_current)

if __name__ == '__main__':
	parser = argparse.ArgumentParser(description=doc)
	parser.add_argument('expected_results', metavar='expected.junit-xml',
			help='junit XML file listing the expected test results.')
	parser.add_argument('current_results', metavar='current.junit-xml',
			help='junit XML file listing the current test results.')

	cmdline = parser.parse_args()
	main(cmdline)