osmotrx: Make sure remote process stops after ssh session is closed
First of all, it was found that vty allocation must be forced (-t -t)
during ssh session creation to make sure SIGHUP is forwarded when
session is closed.
Second, since osmo-trx ignores SIGHUP (osmo_init_ignore_signals()), we
must add a wrapper script which converts received SIGHUP into a SIGINT
to stop osmo-trx.
Change-Id: Ic334a54b1a1827d74fe0b453ac32bb77b8616147
diff --git a/src/osmo_gsm_tester/bts_osmotrx.py b/src/osmo_gsm_tester/bts_osmotrx.py
index 3ed596a..a1814dc 100644
--- a/src/osmo_gsm_tester/bts_osmotrx.py
+++ b/src/osmo_gsm_tester/bts_osmotrx.py
@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
+import stat
import pprint
import tempfile
from abc import ABCMeta, abstractmethod
@@ -142,6 +143,7 @@
CONF_OSMO_TRX = 'osmo-trx.cfg'
REMOTE_DIR = '/osmo-gsm-tester-trx/last_run'
+ WRAPPER_SCRIPT = 'ssh_sigkiller.sh'
##############
# PROTECTED
@@ -207,6 +209,28 @@
proc.launch()
return proc
+ def generate_wrapper_script(self):
+ wrapper_script = self.run_dir.new_file(OsmoTrx.WRAPPER_SCRIPT)
+ with open(wrapper_script, 'w') as f:
+ r = """#!/bin/bash
+ mypid=0
+ sign_handler() {
+ sig=$1
+ echo "received signal handler $sig, killing $mypid"
+ kill $mypid
+ }
+ trap 'sign_handler SIGINT' SIGINT
+ trap 'sign_handler SIGHUP' SIGHUP
+ "$@" &
+ mypid=$!
+ echo "waiting for $mypid"
+ wait $mypid
+ """
+ f.write(r)
+ st = os.stat(wrapper_script)
+ os.chmod(wrapper_script, st.st_mode | stat.S_IEXEC)
+ return wrapper_script
+
##############
# PUBLIC (test API included)
##############
@@ -221,13 +245,21 @@
self.proc_trx = self.launch_process_local(keepalive, self.binary_name(),
'-C', os.path.abspath(self.config_file))
else:
+ # Run remotely through ssh. We need to run osmo-trx under a wrapper
+ # script since osmo-trx ignores SIGHUP and will keep running after
+ # we close local ssh session. The wrapper script catches SIGHUP and
+ # sends SIGINT to it.
+ wrapper_script = self.generate_wrapper_script()
remote_run_dir = util.Dir(OsmoTrx.REMOTE_DIR)
self.remote_inst = process.copy_inst_ssh(self.run_dir, self.inst, remote_run_dir, self.remote_user,
self.listen_ip, self.binary_name(), self.config_file)
+ remote_wrapper_script = remote_run_dir.child(OsmoTrx.WRAPPER_SCRIPT)
remote_config_file = remote_run_dir.child(OsmoTrx.CONF_OSMO_TRX)
remote_lib = self.remote_inst.child('lib')
remote_binary = self.remote_inst.child('bin', self.binary_name())
- args = ('LD_LIBRARY_PATH=%s' % remote_lib, remote_binary, '-C', remote_config_file)
+ process.scp(self.run_dir, self.remote_user, self.listen_ip, 'scp-wrapper-to-remote', wrapper_script, remote_wrapper_script)
+
+ args = ('LD_LIBRARY_PATH=%s' % remote_lib, remote_wrapper_script, remote_binary, '-C', remote_config_file)
self.proc_trx = self.launch_process_remote(self.binary_name(), args, remote_cwd=remote_run_dir, keepalive=keepalive)
def trx_ready(self):
diff --git a/src/osmo_gsm_tester/bts_sysmo.py b/src/osmo_gsm_tester/bts_sysmo.py
index bdf6bb7..66d305a 100644
--- a/src/osmo_gsm_tester/bts_sysmo.py
+++ b/src/osmo_gsm_tester/bts_sysmo.py
@@ -90,6 +90,12 @@
###################
# PUBLIC (test API included)
###################
+ # We get log from ssh stdout instead of usual stderr.
+ def ready_for_pcu(self):
+ if not self.proc_bts or not self.proc_bts.is_running:
+ return False
+ return 'BTS is up' in (self.proc_bts.get_stdout() or '')
+
def start(self, keepalive=False):
if self.bsc is None:
raise RuntimeError('BTS needs to be added to a BSC or NITB before it can be started')
diff --git a/src/osmo_gsm_tester/process.py b/src/osmo_gsm_tester/process.py
index 9db9241..534cdba 100644
--- a/src/osmo_gsm_tester/process.py
+++ b/src/osmo_gsm_tester/process.py
@@ -228,7 +228,10 @@
cd = 'cd "%s"; ' % self.remote_cwd
else:
cd = ''
- self.popen_args = ['ssh', self.remote_user+'@'+self.remote_host,
+ # We need double -t to force tty and be able to forward signals to
+ # processes (SIGHUP) when we close ssh on the local side. As a result,
+ # stderr seems to be merged into stdout in ssh client.
+ self.popen_args = ['ssh', '-t', '-t', self.remote_user+'@'+self.remote_host,
'%s%s' % (cd,
' '.join(self.popen_args))]
self.dbg(self.popen_args, dir=self.run_dir, conf=self.popen_kwargs)