add compare-results.sh, call from start-testsuite.sh
Compare current test results to the expected results, and exit in error on
discrepancies.
Add compare-result.sh: (trivially) grep junit xml output to determine which
tests passed and which didn't, and compare against an expected-result.log,
another junit file from a previous run. Summarize and determine success.
Include an "xfail" feature: tests that are expected to fail are marked as
"xfail", unexpected failures as "FAIL".
In various subdirs, copy the current jenkins jobs' junit xml outputs as
expected-results.log, so that we will start getting useful output in both
jenkins runs and manual local runs.
In start-testsuite.sh, after running the tests, invoke the results comparison.
Due to the single-line parsing nature, the script so far does not distinguish
between error and failure. I doubt that we actually need to do that though.
Related: OS#3136
Change-Id: I87d62a8be73d73a5eeff61a842e7c27a0066079d
diff --git a/bsc/expected-results.log b/bsc/expected-results.log
new file mode 100644
index 0000000..18a9a20
--- /dev/null
+++ b/bsc/expected-results.log
@@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<testsuite name='BSC_Tests' tests='68' failures='4' errors='2' skipped='0' inconc='0' time='932.00'>
+ <testcase classname='BSC_Tests' name='TC_ctrl_msc_connection_status' time='2.267915'/>
+ <testcase classname='BSC_Tests' name='TC_ctrl_msc0_connection_status' time='2.225696'/>
+ <testcase classname='BSC_Tests' name='TC_ctrl' time='4.242800'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_noreply' time='2.229171'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_counter' time='2.226685'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_noest' time='5.225675'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_noreply' time='24.248343'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_ack_est_ind_refused' time='4.231838'/>
+ <testcase classname='BSC_Tests' name='TC_chan_act_nack' time='2.728319'/>
+ <testcase classname='BSC_Tests' name='TC_chan_exhaustion' time='2.239509'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_rll_rel_ind' time='8.276291'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_conn_fail' time='8.290978'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_hard_clear' time='7.431206'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_hard_rlsd' time='4.451855'/>
+ <testcase classname='BSC_Tests' name='TC_chan_rel_a_reset' time='4.437509'/>
+ <testcase classname='BSC_Tests' name='TC_outbound_connect' time='2.230366'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_cic_only' time='2.432230'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_csd' time='2.435962'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_ctm' time='2.431946'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_sign' time='3.139859'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_0' time='5.362074'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1' time='5.360999'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_1_codec_missing' time='3.371507'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_3' time='5.373823'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_fr_a5_4' time='5.385205'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_fr' time='5.370247'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_hr' time='5.370154'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_efr' time='5.373357'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_amr_f' time='5.369318'/>
+ <testcase classname='BSC_Tests' name='TC_assignment_codec_amr_h' time='5.362888'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inact_lchan' time='2.224803'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi1' time='4.226237'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sapi3' time='4.231547'/>
+ <testcase classname='BSC_Tests' name='TC_rll_est_ind_inval_sacch' time='4.246674'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan' time='8.753319'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_nochan' time='8.759194'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_any' time='8.750134'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_sdcch' time='8.754159'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_f' time='8.758687'/>
+ <testcase classname='BSC_Tests' name='TC_paging_tmsi_tch_hf' time='9.289936'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi' time='8.956272'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_ci' time='8.956649'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_ci' time='8.969455'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lai' time='8.763181'>
+ <failure type='fail-verdict'>Timeout expecting { msg_disc := { msg_group := RSL_MDISC_CCHAN (6), transparent := false }, msg_type := RSL_MT_PAGING_CMD (21), ies := { { iei := ?, body := { chan_nr := { u := { ch0 := RSL_CHAN_NR_PCH_AGCH (18) }, tn := ? } } }, { iei := ?, body := { paging_group := ? } }, { iei := ?, body := { ms_identity := { len := ?, payload := ? } } }, * } }
+ BSC_Tests.ttcn:2203 BSC_Tests control part
+ BSC_Tests.ttcn:1131 TC_paging_imsi_nochan_lai testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac' time='8.752368'>
+ <error type='DTE'></error>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_all' time='8.751176'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_plmn_lac_rnc' time='8.563604'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_rnc' time='8.551772'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lac_rnc' time='8.568040'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs' time='8.762731'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_lacs_empty' time='8.548303'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_nochan_cgi_unknown_cid' time='8.557896'/>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_a_reset' time='8.759365'>
+ <failure type='fail-verdict'>Received PAGING after A-RESET
+ BSC_Tests.ttcn:2212 BSC_Tests control part
+ BSC_Tests.ttcn:1306 TC_paging_imsi_a_reset testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_paging_imsi_load' time='12.961436'/>
+ <testcase classname='BSC_Tests' name='TC_paging_counter' time='20.788631'/>
+ <testcase classname='BSC_Tests' name='TC_rsl_drop_counter' time='2.227479'/>
+ <testcase classname='BSC_Tests' name='TC_rsl_unknown_unit_id' time='0.010705'/>
+ <testcase classname='BSC_Tests' name='TC_oml_unknown_unit_id' time='0.010646'/>
+ <testcase classname='BSC_Tests' name='TC_classmark' time='3.185732'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ass_fail' time='8.149850'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ass_compl' time='8.155615'/>
+ <testcase classname='BSC_Tests' name='TC_unsol_ho_fail' time='8.147592'/>
+ <testcase classname='BSC_Tests' name='TC_err_82_short_msg' time='8.161174'/>
+ <testcase classname='BSC_Tests' name='TC_err_84_unknown_msg' time='3.174456'/>
+ <testcase classname='BSC_Tests' name='TC_ho_int' time='7.423814'/>
+ <testcase classname='BSC_Tests' name='TC_bssap_rlsd_does_not_cause_bssmap_reset' time='14.310727'/>
+ <testcase classname='BSC_Tests' name='TC_bssmap_clear_does_not_cause_bssmap_reset' time='45.459198'>
+ <failure type='fail-verdict'>Timeout of T_guard
+ BSC_Tests.ttcn:2230 BSC_Tests control part
+ BSC_Tests.ttcn:2056 TC_bssmap_clear_does_not_cause_bssmap_reset testcase
+ </failure>
+ </testcase>
+ <testcase classname='BSC_Tests' name='TC_ms_rel_ind_does_not_cause_bssmap_reset' time='45.320202'>
+ <failure type='fail-verdict'>Timeout of T_guard
+ BSC_Tests.ttcn:2231 BSC_Tests control part
+ BSC_Tests.ttcn:2115 TC_ms_rel_ind_does_not_cause_bssmap_reset testcase
+ </failure>
+ </testcase>
+</testsuite>