resource: Support waiting for reserved resources until available

Before this patch, almost everything was in place to support concurrent
osmo-gsm-tester instances sharing a common state dir. However, during
resource reservation, if the reservation couldn't be done due to too
many resources being in use, osmo-gsm-tester would fail and skip the
test suite.
With this patch, OGT will wait until some reserved resources are
released and then try requesting the reservation again.

Change-Id: I938602ee890712fda82fd3f812d8edb1bcd05e08
diff --git a/selftest/resource_test/resource_test.ok b/selftest/resource_test/resource_test.ok
index 91acaaf..5f7d5f6 100644
--- a/selftest/resource_test/resource_test.ok
+++ b/selftest/resource_test/resource_test.ok
@@ -116,6 +116,59 @@
 *** end: all resources
 
 - request some resources
+--- testowner: Verifying 2 x arfcn (candidates: 10)
+--- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
+  arfcn: '512'
+  band: GSM-1800
+- _hash: 022621e513c5a5bf33b77430a1e9c886be676fa1
+  arfcn: '514'
+  band: GSM-1800
+--- testowner: Verifying 2 x bts (candidates: 3)
+--- testowner: DBG: Picked - _hash: d2aa7c1124943de352351b650ca0c751784da6b6
+  addr: 10.42.42.114
+  band: GSM-1800
+  ciphers:
+  - a5_0
+  - a5_1
+  direct_pcu: 'True'
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: osmo-bts-sysmo
+- _hash: 2158317d5e0055070e7174c2498dedf53a2957e9
+  addr: 10.42.42.50
+  band: GSM-1800
+  ciphers:
+  - a5_0
+  - a5_1
+  ipa_unit_id: '6'
+  label: Ettus B200
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'True'
+  type: osmo-bts-trx
+--- testowner: Verifying 1 x ip_address (candidates: 5)
+--- testowner: DBG: Picked - _hash: fd103b22c7cf2480d609150e06f4bbd92ac78d8c
+  addr: 10.42.42.2
+--- testowner: Verifying 2 x modem (candidates: 4)
+--- testowner: DBG: Picked - _hash: 0b538cb6ad799fbd7c2953fd3b4463a76c7cc9c0
+  auth_algo: comp128v1
+  ciphers:
+  - a5_0
+  - a5_1
+  imsi: '901700000009031'
+  ki: 80A37E6FDEA931EAC92FFA5F671EFEAD
+  label: sierra_1
+  path: /sierra_1
+- _hash: 3a6e7747dfe7dfdf817bd3351031bd08051605c3
+  auth_algo: comp128v1
+  ciphers:
+  - a5_0
+  - a5_1
+  imsi: '901700000009029'
+  ki: 00969E283349D354A8239E877F2E0866
+  label: sierra_2
+  path: /sierra_2
+--- testowner: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/resource_test/conf/test_work/state_dir
 --- testowner: Reserving 2 x arfcn (candidates: 10)
 --- testowner: DBG: Picked - _hash: e620569450f8259b3f0212ec19c285dd07df063c
   arfcn: '512'
@@ -168,6 +221,7 @@
   ki: 00969E283349D354A8239E877F2E0866
   label: sierra_2
   path: /sierra_2
+--- testowner: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/resource_test/conf/test_work/state_dir, is_recursive=False>
 ~~~ currently reserved:
 arfcn:
 - _hash: e620569450f8259b3f0212ec19c285dd07df063c
@@ -290,3 +344,27 @@
 3rd subset should not match, pass
 3rd subset should not match, pass
 4th subset should not match, pass
+*** concurrent allocation:
+--- testowner1: Verifying 2 x arfcn (candidates: 10)
+--- testowner1: Verifying 2 x bts (candidates: 3)
+--- testowner1: Verifying 1 x ip_address (candidates: 5)
+--- testowner1: Verifying 2 x modem (candidates: 4)
+--- testowner1: Reserving 2 x arfcn (candidates: 10)
+--- testowner1: Reserving 2 x bts (candidates: 3)
+--- testowner1: Reserving 1 x ip_address (candidates: 5)
+--- testowner1: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() start
+--- testowner2: Verifying 2 x arfcn (candidates: 10)
+--- testowner2: Verifying 2 x bts (candidates: 3)
+--- testowner2: Verifying 1 x ip_address (candidates: 5)
+--- testowner2: Verifying 2 x modem (candidates: 4)
+--- testowner2: Reserving 2 x arfcn (candidates: 8)
+--- testowner2: Reserving 2 x bts (candidates: 1)
+--- testowner2: Unable to reserve resources, too many currently reserved. Waiting until some are available again
+- 1st instance free()
+--- testowner2: Reserving 2 x arfcn (candidates: 10)
+--- testowner2: Reserving 2 x bts (candidates: 3)
+--- testowner2: Reserving 1 x ip_address (candidates: 5)
+--- testowner2: Reserving 2 x modem (candidates: 4)
+- 2nd instance reserve() done
+*** end: concurrent allocation
diff --git a/selftest/resource_test/resource_test.py b/selftest/resource_test/resource_test.py
index f18aa73..13cce97 100755
--- a/selftest/resource_test/resource_test.py
+++ b/selftest/resource_test/resource_test.py
@@ -6,6 +6,8 @@
 import pprint
 import shutil
 import atexit
+import time
+import threading
 import _prep
 from osmo_gsm_tester.core import config, log, util, resource
 from osmo_gsm_tester.core.schema import generate_schemas
@@ -128,4 +130,26 @@
 if not resource.item_matches(superset, subset):
     print('4th subset should not match, pass')
 
+print('*** concurrent allocation:')
+origin1 = log.Origin(None, 'testowner1')
+origin2 = log.Origin(None, 'testowner2')
+# We disable dbg() for second thread since FileWatch output result is
+# non-deterministic, since sometimes 1 Modiffied event is triggered, sometimes 2.
+origin1.dbg = origin2.dbg = lambda obj, *messages, _src=3, **named_items: None
+resources2 = None
+def second_ogt_instance():
+    # should block here until "resources" are freed.
+    print('- 2nd instance reserve() start')
+    resources2 = pool.reserve(origin2, config.replicate_times(want), config.replicate_times(modifiers))
+    print('- 2nd instance reserve() done')
+    resources2.free()
+resources = pool.reserve(origin1, config.replicate_times(want), config.replicate_times(modifiers))
+th = threading.Thread(target=second_ogt_instance)
+th.start()
+time.sleep(1.0)
+print('- 1st instance free()')
+resources.free()
+th.join()
+print('*** end: concurrent allocation')
+
 # vim: expandtab tabstop=4 shiftwidth=4
diff --git a/selftest/suite_test/suite_test.ok b/selftest/suite_test/suite_test.ok
index 58593fd..150a4e9 100644
--- a/selftest/suite_test/suite_test.ok
+++ b/selftest/suite_test/suite_test.ok
@@ -43,6 +43,57 @@
 tst {combining_scenarios='resources'}: DBG: {definition_conf={bts=[{'label': 'sysmoCell 5000'}, {'label': 'sysmoCell 5000'}, {'type': 'sysmo'}], ip_address=[{}], modem=[{}, {}]}}  [test_suite↪{combining_scenarios='resources'}]
 tst test_suite: DBG: {combining='modifiers'}
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [test_suite↪{combining_scenarios='modifiers'}]
+tst test_suite: Verifying 3 x bts (candidates: 6)
+tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - max_power_red: '3'
+    nominal_power: '10'
+  - max_power_red: '0'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - nominal_power: '10'
+  - max_power_red: '1'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9
+  addr: 10.42.42.114
+  band: GSM-1800
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: sysmo
+tst test_suite: Verifying 1 x ip_address (candidates: 3)
+tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4
+  addr: 10.42.42.1
+tst test_suite: Verifying 2 x modem (candidates: 16)
+tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f
+  imsi: '901700000007801'
+  ki: D620F48487B1B782DA55DF6717F08FF9
+  label: m7801
+  path: /wavecom_0
+- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254
+  imsi: '901700000007802'
+  ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
+  label: m7802
+  path: /wavecom_1
+tst test_suite: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir
 tst test_suite: Reserving 3 x bts (candidates: 6)
 tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
   addr: 10.42.42.53
@@ -93,6 +144,7 @@
   ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
   label: m7802
   path: /wavecom_1
+tst test_suite: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>
 
 ----------------------------------------------
 trial test_suite hello_world.py
@@ -204,6 +256,60 @@
 tst test_suite: DBG: {combining='modifiers'}  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [test_suite↪{combining_scenarios='modifiers'}]  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers', scenario='foo'}: DBG: {conf={}, scenario='foo'}  [test_suite↪{combining_scenarios='modifiers', scenario='foo'}]  [suite.py:[LINENR]]
+tst test_suite: Verifying 3 x bts (candidates: 6)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - max_power_red: '3'
+    nominal_power: '10'
+  - max_power_red: '0'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - nominal_power: '10'
+  - max_power_red: '1'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9
+  addr: 10.42.42.114
+  band: GSM-1800
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: sysmo
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4
+  addr: 10.42.42.1
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 2 x modem (candidates: 16)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f
+  imsi: '901700000007801'
+  ki: D620F48487B1B782DA55DF6717F08FF9
+  label: m7801
+  path: /wavecom_0
+- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254
+  imsi: '901700000007802'
+  ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
+  label: m7802
+  path: /wavecom_1
+  [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir  [util.py:[LINENR]]
 tst test_suite: Reserving 3 x bts (candidates: 6)  [resource.py:[LINENR]]
 tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
   addr: 10.42.42.53
@@ -257,6 +363,7 @@
   label: m7802
   path: /wavecom_1
   [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>  [util.py:[LINENR]]
 
 ----------------------------------------------
 trial test_suite hello_world.py
@@ -294,6 +401,60 @@
 tst test_suite: DBG: {combining='modifiers'}  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [test_suite↪{combining_scenarios='modifiers'}]  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers', scenario='foo'}: DBG: {conf={}, scenario='foo'}  [test_suite↪{combining_scenarios='modifiers', scenario='foo'}]  [suite.py:[LINENR]]
+tst test_suite: Verifying 3 x bts (candidates: 6)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - max_power_red: '3'
+    nominal_power: '10'
+  - max_power_red: '0'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - nominal_power: '10'
+  - max_power_red: '1'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9
+  addr: 10.42.42.114
+  band: GSM-1800
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: sysmo
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4
+  addr: 10.42.42.1
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 2 x modem (candidates: 16)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f
+  imsi: '901700000007801'
+  ki: D620F48487B1B782DA55DF6717F08FF9
+  label: m7801
+  path: /wavecom_0
+- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254
+  imsi: '901700000007802'
+  ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
+  label: m7802
+  path: /wavecom_1
+  [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir  [util.py:[LINENR]]
 tst test_suite: Reserving 3 x bts (candidates: 6)  [resource.py:[LINENR]]
 tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
   addr: 10.42.42.53
@@ -347,6 +508,7 @@
   label: m7802
   path: /wavecom_1
   [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>  [util.py:[LINENR]]
 
 ----------------------------------------------
 trial test_suite hello_world.py
@@ -380,6 +542,60 @@
 tst test_suite: DBG: {combining='modifiers'}  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [test_suite↪{combining_scenarios='modifiers'}]  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers', scenario='foo'}: DBG: {conf={bts=[{'trx_list': [{'nominal_power': '20'}, {'nominal_power': '20'}]}, {'trx_list': [{'nominal_power': '20'}, {'nominal_power': '20'}]}, {'type': 'sysmo'}]}, scenario='foo'}  [test_suite↪{combining_scenarios='modifiers', scenario='foo'}]  [suite.py:[LINENR]]
+tst test_suite: Verifying 3 x bts (candidates: 6)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - max_power_red: '3'
+    nominal_power: '10'
+  - max_power_red: '0'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - nominal_power: '10'
+  - max_power_red: '1'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9
+  addr: 10.42.42.114
+  band: GSM-1800
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: sysmo
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4
+  addr: 10.42.42.1
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 2 x modem (candidates: 16)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f
+  imsi: '901700000007801'
+  ki: D620F48487B1B782DA55DF6717F08FF9
+  label: m7801
+  path: /wavecom_0
+- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254
+  imsi: '901700000007802'
+  ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
+  label: m7802
+  path: /wavecom_1
+  [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir  [util.py:[LINENR]]
 tst test_suite: Reserving 3 x bts (candidates: 6)  [resource.py:[LINENR]]
 tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
   addr: 10.42.42.53
@@ -433,6 +649,7 @@
   label: m7802
   path: /wavecom_1
   [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>  [util.py:[LINENR]]
 resources(test_suite)={'bts': [{'_hash': 'a59640b8ba6a373552b24a6f9f65cadd2347bace',
           '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]',
           'addr': '10.42.42.53',
@@ -516,6 +733,60 @@
 tst test_suite: DBG: {combining='modifiers'}  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [test_suite↪{combining_scenarios='modifiers'}]  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers', scenario='foo'}: DBG: {conf={}, scenario='foo'}  [test_suite↪{combining_scenarios='modifiers', scenario='foo'}]  [suite.py:[LINENR]]
+tst test_suite: Verifying 3 x bts (candidates: 6)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - max_power_red: '3'
+    nominal_power: '10'
+  - max_power_red: '0'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: c2feabd082c36a1cdeccb9a5237dfff7dbadb009
+  addr: 10.42.42.53
+  band: GSM-1800
+  ipa_unit_id: '7'
+  label: sysmoCell 5000
+  osmo_trx:
+    clock_reference: external
+    launch_trx: 'False'
+    trx_ip: 10.42.42.112
+  trx_list:
+  - nominal_power: '10'
+  - max_power_red: '1'
+    nominal_power: '12'
+  type: osmo-bts-trx
+- _hash: 07d9c8aaa940b674efcbbabdd69f58a6ce4e94f9
+  addr: 10.42.42.114
+  band: GSM-1800
+  ipa_unit_id: '1'
+  label: sysmoBTS 1002
+  type: sysmo
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: cde1debf28f07f94f92c761b4b7c6bf35785ced4
+  addr: 10.42.42.1
+  [resource.py:[LINENR]]
+tst test_suite: Verifying 2 x modem (candidates: 16)  [resource.py:[LINENR]]
+tst test_suite: DBG: Picked - _hash: 19c69e45aa090fb511446bd00797690aa82ff52f
+  imsi: '901700000007801'
+  ki: D620F48487B1B782DA55DF6717F08FF9
+  label: m7801
+  path: /wavecom_0
+- _hash: e1a46516a1fb493b2617ab14fc1693a9a45ec254
+  imsi: '901700000007802'
+  ki: 47FDB2D55CE6A10A85ABDAD034A5B7B3
+  label: m7802
+  path: /wavecom_1
+  [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir  [util.py:[LINENR]]
 tst test_suite: Reserving 3 x bts (candidates: 6)  [resource.py:[LINENR]]
 tst test_suite: DBG: Picked - _hash: a59640b8ba6a373552b24a6f9f65cadd2347bace
   addr: 10.42.42.53
@@ -569,6 +840,7 @@
   label: m7802
   path: /wavecom_1
   [resource.py:[LINENR]]
+tst test_suite: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>  [util.py:[LINENR]]
 resources(test_suite)={'bts': [{'_hash': 'a59640b8ba6a373552b24a6f9f65cadd2347bace',
           '_reserved_by': 'test_suite-[ID_NUM]-[ID_NUM]',
           'addr': '10.42.42.53',
@@ -662,10 +934,16 @@
 tst {combining_scenarios='resources'}: DBG: {definition_conf={ip_address=[{'addr': '10.42.42.2'}]}}  [suiteC↪{combining_scenarios='resources'}]  [suite.py:[LINENR]]
 tst suiteC: DBG: {combining='modifiers'}  [suite.py:[LINENR]]
 tst {combining_scenarios='modifiers'}: DBG: {definition_conf={}}  [suiteC↪{combining_scenarios='modifiers'}]  [suite.py:[LINENR]]
+tst suiteC: Verifying 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
+tst suiteC: DBG: Picked - _hash: fd103b22c7cf2480d609150e06f4bbd92ac78d8c
+  addr: 10.42.42.2
+  [resource.py:[LINENR]]
+tst suiteC: DBG: FileWatch: scheduling watch for directory [PATH]/selftest/suite_test/test_work/state_dir  [util.py:[LINENR]]
 tst suiteC: Reserving 1 x ip_address (candidates: 3)  [resource.py:[LINENR]]
 tst suiteC: DBG: Picked - _hash: fd103b22c7cf2480d609150e06f4bbd92ac78d8c
   addr: 10.42.42.2
   [resource.py:[LINENR]]
+tst suiteC: DBG: FileWatch: unscheduling watch <ObservedWatch: path=[PATH]/selftest/suite_test/test_work/state_dir, is_recursive=False>  [util.py:[LINENR]]
 
 ----------------------------------------------
 trial suiteC test_template_overlay.py