add compare-results.sh, call from start-testsuite.sh
Compare current test results to the expected results, and exit in error on
discrepancies.
Add compare-result.sh: (trivially) grep junit xml output to determine which
tests passed and which didn't, and compare against an expected-result.log,
another junit file from a previous run. Summarize and determine success.
Include an "xfail" feature: tests that are expected to fail are marked as
"xfail", unexpected failures as "FAIL".
In various subdirs, copy the current jenkins jobs' junit xml outputs as
expected-results.log, so that we will start getting useful output in both
jenkins runs and manual local runs.
In start-testsuite.sh, after running the tests, invoke the results comparison.
Due to the single-line parsing nature, the script so far does not distinguish
between error and failure. I doubt that we actually need to do that though.
Related: OS#3136
Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
diff --git a/msc/expected-results.log b/msc/expected-results.log
new file mode 100644
index 0000000..3945a5f
--- /dev/null
+++ b/msc/expected-results.log
@@ -0,0 +1,110 @@
+<?xml version="1.0"?>
+<testsuite name='Titan' tests='45' failures='12' errors='1' skipped='0' inconc='0' time='497.00'>
+ <testcase classname='MSC_Tests' name='TC_cr_before_reset' time='4.009127'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_tmsi' time='1.035697'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_noauth_notmsi' time='1.045502'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_reject' time='6.041282'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1829 MSC_Tests control part
+ MSC_Tests.ttcn:502 TC_lu_imsi_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_timeout_gsup' time='11.044559'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1830 MSC_Tests control part
+ MSC_Tests.ttcn:531 TC_lu_imsi_timeout_gsup testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi' time='1.036704'/>
+ <testcase classname='MSC_Tests' name='TC_cmserv_imsi_unknown' time='1.021692'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mo_call' time='4.291881'/>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_sai_timeout' time='6.032964'/>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_sai_err' time='1.028503'/>
+ <testcase classname='MSC_Tests' name='TC_lu_clear_request' time='2.029726'/>
+ <testcase classname='MSC_Tests' name='TC_lu_disconnect' time='2.028908'/>
+ <testcase classname='MSC_Tests' name='TC_lu_by_imei' time='6.030599'/>
+ <testcase classname='MSC_Tests' name='TC_lu_by_tmsi_noauth_unknown' time='1.246818'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_imsi' time='1.238427'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_tmsi' time='1.228194'/>
+ <testcase classname='MSC_Tests' name='TC_imsi_detach_by_imei' time='1.233577'/>
+ <testcase classname='MSC_Tests' name='TC_emerg_call_imei_reject' time='6.025962'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1843 MSC_Tests control part
+ MSC_Tests.ttcn:903 TC_emerg_call_imei_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_emerg_call_imsi' time='4.295068'/>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_vgcs_reject' time='6.251082'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1845 MSC_Tests control part
+ MSC_Tests.ttcn:940 TC_cm_serv_req_vgcs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_vbs_reject' time='6.249901'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1846 MSC_Tests control part
+ MSC_Tests.ttcn:961 TC_cm_serv_req_vbs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_serv_req_lcs_reject' time='6.244740'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1847 MSC_Tests control part
+ MSC_Tests.ttcn:982 TC_cm_serv_req_lcs_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_cm_reest_req_reject' time='6.250134'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1848 MSC_Tests control part
+ MSC_Tests.ttcn:1003 TC_cm_reest_req_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_auth_2G_fail' time='1.042594'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_13' time='1.052763'/>
+ <testcase classname='MSC_Tests' name='TC_cl3_no_payload' time='1.021188'/>
+ <testcase classname='MSC_Tests' name='TC_cl3_rnd_payload' time='5.996049'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand or SCCP Release
+ MSC_Tests.ttcn:1852 MSC_Tests control part
+ MSC_Tests.ttcn:1125 TC_cl3_rnd_payload testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_establish_and_nothing' time='6.263255'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1853 MSC_Tests control part
+ MSC_Tests.ttcn:1142 TC_establish_and_nothing testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_setup_and_nothing' time='31.481429'>
+ <failure type='fail-verdict'>Timeout waiting for ClearCommand/Release
+ MSC_Tests.ttcn:1854 MSC_Tests control part
+ MSC_Tests.ttcn:1166 TC_mo_setup_and_nothing testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_crcx_ran_timeout' time='5.267724'/>
+ <testcase classname='MSC_Tests' name='TC_mo_crcx_ran_reject' time='31.279836'>
+ <failure type='fail-verdict'>Timeout waiting for channel release
+ MSC_Tests.ttcn:1856 MSC_Tests control part
+ MSC_Tests.ttcn:1283 TC_mo_crcx_ran_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mt_crcx_ran_reject' time='31.062826'>
+ <failure type='fail-verdict'>Timeout waiting for channel release
+ MSC_Tests.ttcn:1857 MSC_Tests control part
+ MSC_Tests.ttcn:1380 TC_mt_crcx_ran_reject testcase
+ </failure>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_mo_setup_and_dtmf_dup' time='2.468132'/>
+ <testcase classname='MSC_Tests' name='TC_gsup_cancel' time='1.259044'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_1_13' time='1.049565'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_13' time='1.054199'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1' time='1.034815'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_no_cm' time='1.038335'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_13_2' time='1.042416'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_013_2' time='1.050732'/>
+ <testcase classname='MSC_Tests' name='TC_mo_release_timeout' time='25.264303'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mt_call_no_dlcx_resp' time='10.273413'/>
+ <testcase classname='MSC_Tests' name='TC_reset_two' time='2.999986'/>
+ <testcase classname='MSC_Tests' name='TC_lu_and_mt_call' time='4.284560'/>
+ <testcase classname='MSC_Tests' name='TC_lu_imsi_auth_tmsi_encr_3_1_log_msc_debug' time='1.040196'/>
+</testsuite>