resource: Support waiting for reserved resources until available
Before this patch, almost everything was in place to support concurrent
osmo-gsm-tester instances sharing a common state dir. However, during
resource reservation, if the reservation couldn't be done due to too
many resources being in use, osmo-gsm-tester would fail and skip the
test suite.
With this patch, OGT will wait until some reserved resources are
released and then try requesting the reservation again.
Change-Id: I938602ee890712fda82fd3f812d8edb1bcd05e08
diff --git a/selftest/resource_test/resource_test.ok b/selftest/resource_test/resource_test.ok
index 91acaaf..5f7d5f6 100644
--- a/selftest/resource_test/resource_test.ok
+++ b/selftest/resource_test/resource_test.ok
@@ -116,6 +116,59 @@
*** end: all resources
- request some resources
+--- testowner: Verifying 2 x arfcn (candidates: 10)
+--- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
+ arfcn: '512'
+ band: GSM-1800
+- _hash: 022621e513c5a5bf33b77430a1e9c886be676fa1
+ arfcn: '514'
+ band: GSM-1800
+--- testowner: Verifying 2 x bts (candidates: 3)
+--- testowner: DBG: Picked - _hash: d2aa7c1124943de352351b650ca0c751784da6b6
+ addr: 10.42.42.114
+ band: GSM-1800
+ ciphers:
+ - a5_0
+ - a5_1
+ direct_pcu: 'True'
+ ipa_unit_id: '1'
+ label: sysmoBTS 1002
+ type: osmo-bts-sysmo
+- _hash: 2158317d5e0055070e7174c2498dedf53a2957e9
+ addr: 10.42.42.50
+ band: GSM-1800
+ ciphers:
+ - a5_0
+ - a5_1
+ ipa_unit_id: '6'
+ label: Ettus B200
+ osmo_trx:
+ clock_reference: external
+ launch_trx: 'True'
+ type: osmo-bts-trx
+--- testowner: Verifying 1 x ip_address (candidates: 5)
+--- testowner: DBG: Picked - _hash: fd103b22c7cf2480d609150e06f4bbd92ac78d8c
+ addr: 10.42.42.2
+--- testowner: Verifying 2 x modem (candidates: 4)
+--- testowner: DBG: Picked - _hash: 0b538cb6ad799fbd7c2953fd3b4463a76c7cc9c0
+ auth_algo: comp128v1
+ ciphers:
+ - a5_0
+ - a5_1
+ imsi: '901700000009031'
+ ki: 80A37E6FDEA931EAC92FFA5F671EFEAD
+ label: sierra_1
+ path: /sierra_1
+- _hash: 3a6e7747dfe7dfdf817bd3351031bd08051605c3
+ auth_algo: comp128v1
+ ciphers:
+ - a5_0
+ - a5_1
+ imsi: '901700000009029'
+ ki: 00969E283349D354A8239E877F2E0866
+ label: sierra_2
+ path: /sierra_2
+--- testowner: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/resource_test/conf/test_work/state_dir
--- testowner: Reserving 2 x arfcn (candidates: 10)
--- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
arfcn: '512'
@@ -168,6 +221,7 @@
ki: 00969E283349D354A8239E877F2E0866
label: sierra_2
path: /sierra_2
+--- testowner: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/resource_test/conf/test_work/state_dir, is_recursive=False>
~~~ currently reserved:
arfcn:
- _hash: e620569450f8259b3f0212ec19c285dd07df063c
@@ -290,3 +344,27 @@
3rd subset should not match, pass
3rd subset should not match, pass
4th subset should not match, pass
+*** concurrent allocation:
+--- testowner1: Verifying 2 x arfcn (candidates: 10)
+--- testowner1: Verifying 2 x bts (candidates: 3)
+--- testowner1: Verifying 1 x ip_address (candidates: 5)
+--- testowner1: Verifying 2 x modem (candidates: 4)
+--- testowner1: Reserving 2 x arfcn (candidates: 10)
+--- testowner1: Reserving 2 x bts (candidates: 3)
+--- testowner1: Reserving 1 x ip_address (candidates: 5)
+--- testowner1: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() start
+--- testowner2: Verifying 2 x arfcn (candidates: 10)
+--- testowner2: Verifying 2 x bts (candidates: 3)
+--- testowner2: Verifying 1 x ip_address (candidates: 5)
+--- testowner2: Verifying 2 x modem (candidates: 4)
+--- testowner2: Reserving 2 x arfcn (candidates: 8)
+--- testowner2: Reserving 2 x bts (candidates: 1)
+--- testowner2: Unable to reserve resources, too many currently reserved. Waiting until some are available again
+- 1st instance free()
+--- testowner2: Reserving 2 x arfcn (candidates: 10)
+--- testowner2: Reserving 2 x bts (candidates: 3)
+--- testowner2: Reserving 1 x ip_address (candidates: 5)
+--- testowner2: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() done
+*** end: concurrent allocation
diff --git a/selftest/resource_test/resource_test.py b/selftest/resource_test/resource_test.py
index f18aa73..13cce97 100755
--- a/selftest/resource_test/resource_test.py
+++ b/selftest/resource_test/resource_test.py
@@ -6,6 +6,8 @@
import pprint
import shutil
import atexit
+import time
+import threading
import _prep
from osmo_gsm_tester.core import config, log, util, resource
from osmo_gsm_tester.core.schema import generate_schemas
@@ -128,4 +130,26 @@
if not resource.item_matches(superset, subset):
print('4th subset should not match, pass')
+print('*** concurrent allocation:')
+origin1 = log.Origin(None, 'testowner1')
+origin2 = log.Origin(None, 'testowner2')
+# We disable dbg() for second thread since FileWatch output result is
+# non-deterministic, since sometimes 1 Modiffied event is triggered, sometimes 2.
+origin1.dbg = origin2.dbg = lambda obj, *messages, _src=3, **named_items: None
+resources2 = None
+def second_ogt_instance():
+ # should block here until "resources" are freed.
+ print('- 2nd instance reserve() start')
+ resources2 = pool.reserve(origin2, config.replicate_times(want), config.replicate_times(modifiers))
+ print('- 2nd instance reserve() done')
+ resources2.free()
+resources = pool.reserve(origin1, config.replicate_times(want), config.replicate_times(modifiers))
+th = threading.Thread(target=second_ogt_instance)
+th.start()
+time.sleep(1.0)
+print('- 1st instance free()')
+resources.free()
+th.join()
+print('*** end: concurrent allocation')
+
# vim: expandtab tabstop=4 shiftwidth=4