Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 1 | # osmo_gsm_tester: read and manage config files and global config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 2 | # |
| 3 | # Copyright (C) 2016-2017 by sysmocom - s.f.m.c. GmbH |
| 4 | # |
| 5 | # Author: Neels Hofmeyr <neels@hofmeyr.de> |
| 6 | # |
| 7 | # This program is free software: you can redistribute it and/or modify |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 8 | # it under the terms of the GNU General Public License as |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 9 | # published by the Free Software Foundation, either version 3 of the |
| 10 | # License, or (at your option) any later version. |
| 11 | # |
| 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 15 | # GNU General Public License for more details. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 16 | # |
Harald Welte | 2720534 | 2017-06-03 09:51:45 +0200 | [diff] [blame] | 17 | # You should have received a copy of the GNU General Public License |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 19 | |
| 20 | # discussion for choice of config file format: |
| 21 | # |
| 22 | # Python syntax is insane, because it allows the config file to run arbitrary |
| 23 | # python commands. |
| 24 | # |
| 25 | # INI file format is nice and simple, but it doesn't allow having the same |
| 26 | # section numerous times (e.g. to define several modems or BTS models) and does |
| 27 | # not support nesting. |
| 28 | # |
| 29 | # JSON has too much braces and quotes to be easy to type |
| 30 | # |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 31 | # YAML formatting is lean, but: |
| 32 | # - too powerful. The normal load() allows arbitrary code execution. There is |
| 33 | # safe_load(). |
| 34 | # - allows several alternative ways of formatting, better to have just one |
| 35 | # authoritative style. |
| 36 | # - tries to detect types. It would be better to receive every setting as |
| 37 | # simple string rather than e.g. an IMSI as an integer. |
| 38 | # - e.g. an IMSI starting with a zero is interpreted as octal value, resulting |
| 39 | # in super confusing error messages if the user merely forgets to quote it. |
| 40 | # - does not tell me which line a config item came from, so no detailed error |
| 41 | # message is possible. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 42 | # |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 43 | # The Python ConfigParserShootout page has numerous contestants, but many of |
| 44 | # those seem to be not widely used / standardized or even tested. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 45 | # https://wiki.python.org/moin/ConfigParserShootout |
| 46 | # |
| 47 | # The optimum would be a stripped down YAML format. |
| 48 | # In the lack of that, we shall go with yaml.load_safe() + a round trip |
| 49 | # (feeding back to itself), converting keys to lowercase and values to string. |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 50 | # There is no solution for octal interpretations nor config file source lines |
| 51 | # unless, apparently, we implement our own config parser. |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 52 | |
| 53 | import yaml |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 54 | import os |
Pau Espin Pedrol | 802dfe5 | 2017-09-12 13:43:40 +0200 | [diff] [blame] | 55 | import copy |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 56 | |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 57 | from . import log, schema, util, template |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 58 | from .util import is_dict, is_list, Dir, get_tempdir |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 59 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 60 | ENV_PREFIX = 'OSMO_GSM_TESTER_' |
| 61 | ENV_CONF = os.getenv(ENV_PREFIX + 'CONF') |
| 62 | |
Neels Hofmeyr | f15eaf9 | 2017-06-05 18:03:53 +0200 | [diff] [blame] | 63 | override_conf = None |
| 64 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 65 | DEFAULT_CONFIG_LOCATIONS = [ |
| 66 | '.', |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 67 | os.path.join(os.getenv('HOME'), '.config', 'osmo-gsm-tester'), |
| 68 | '/usr/local/etc/osmo-gsm-tester', |
| 69 | '/etc/osmo-gsm-tester' |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 70 | ] |
| 71 | |
| 72 | PATHS_CONF = 'paths.conf' |
Neels Hofmeyr | d46ea13 | 2017-04-08 15:56:31 +0200 | [diff] [blame] | 73 | DEFAULT_SUITES_CONF = 'default-suites.conf' |
Pau Espin Pedrol | a2e81f5 | 2020-03-11 20:07:23 +0100 | [diff] [blame] | 74 | DEFAULTS_CONF = 'defaults.conf' |
Pau Espin Pedrol | 66a3891 | 2020-03-11 20:11:08 +0100 | [diff] [blame] | 75 | RESOURCES_CONF = 'resources.conf' |
| 76 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 77 | PATH_STATE_DIR = 'state_dir' |
| 78 | PATH_SUITES_DIR = 'suites_dir' |
| 79 | PATH_SCENARIOS_DIR = 'scenarios_dir' |
| 80 | PATHS_SCHEMA = { |
| 81 | PATH_STATE_DIR: schema.STR, |
| 82 | PATH_SUITES_DIR: schema.STR, |
| 83 | PATH_SCENARIOS_DIR: schema.STR, |
| 84 | } |
| 85 | |
| 86 | PATHS_TEMPDIR_STR = '$TEMPDIR' |
| 87 | |
| 88 | PATHS = None |
| 89 | |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 90 | def _get_config_file(basename, fail_if_missing=True): |
Neels Hofmeyr | f15eaf9 | 2017-06-05 18:03:53 +0200 | [diff] [blame] | 91 | if override_conf: |
| 92 | locations = [ override_conf ] |
| 93 | elif ENV_CONF: |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 94 | locations = [ ENV_CONF ] |
| 95 | else: |
| 96 | locations = DEFAULT_CONFIG_LOCATIONS |
| 97 | |
| 98 | for l in locations: |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 99 | real_l = os.path.realpath(l) |
| 100 | p = os.path.realpath(os.path.join(real_l, basename)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 101 | if os.path.isfile(p): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 102 | log.dbg('Found config file', basename, 'as', p, 'in', l, 'which is', real_l, _category=log.C_CNF) |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 103 | return (p, real_l) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 104 | if not fail_if_missing: |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 105 | return None, None |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 106 | raise RuntimeError('configuration file not found: %r in %r' % (basename, |
| 107 | [os.path.abspath(p) for p in locations])) |
| 108 | |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 109 | def get_config_file(basename, fail_if_missing=True): |
| 110 | path, found_in = _get_config_file(basename, fail_if_missing) |
| 111 | return path |
| 112 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 113 | def read_config_file(basename, validation_schema=None, if_missing_return=False): |
| 114 | fail_if_missing = True |
| 115 | if if_missing_return is not False: |
| 116 | fail_if_missing = False |
| 117 | path = get_config_file(basename, fail_if_missing=fail_if_missing) |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 118 | if path is None: |
| 119 | return if_missing_return |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 120 | return read(path, validation_schema=validation_schema, if_missing_return=if_missing_return) |
| 121 | |
| 122 | def get_configured_path(label, allow_unset=False): |
| 123 | global PATHS |
| 124 | |
| 125 | env_name = ENV_PREFIX + label.upper() |
| 126 | env_path = os.getenv(env_name) |
| 127 | if env_path: |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 128 | real_env_path = os.path.realpath(env_path) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 129 | log.dbg('Found path', label, 'as', env_path, 'in', '$' + env_name, 'which is', real_env_path, _category=log.C_CNF) |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 130 | return real_env_path |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 131 | |
| 132 | if PATHS is None: |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 133 | paths_file, found_in = _get_config_file(PATHS_CONF) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 134 | PATHS = read(paths_file, PATHS_SCHEMA) |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 135 | # sorted for deterministic regression test results |
| 136 | for key, path in sorted(PATHS.items()): |
Your Name | 3c6673a | 2017-04-08 18:52:39 +0200 | [diff] [blame] | 137 | if not path.startswith(os.pathsep): |
Neels Hofmeyr | ef9ed2d | 2017-05-04 16:39:29 +0200 | [diff] [blame] | 138 | PATHS[key] = os.path.realpath(os.path.join(found_in, path)) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 139 | log.dbg(paths_file + ': relative path', path, 'is', PATHS[key], _category=log.C_CNF) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 140 | p = PATHS.get(label) |
| 141 | if p is None and not allow_unset: |
| 142 | raise RuntimeError('missing configuration in %s: %r' % (PATHS_CONF, label)) |
| 143 | |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 144 | log.dbg('Found path', label, 'as', p, _category=log.C_CNF) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 145 | if p.startswith(PATHS_TEMPDIR_STR): |
| 146 | p = os.path.join(get_tempdir(), p[len(PATHS_TEMPDIR_STR):]) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 147 | log.dbg('Path', label, 'contained', PATHS_TEMPDIR_STR, 'and becomes', p, _category=log.C_CNF) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 148 | return p |
| 149 | |
| 150 | def get_state_dir(): |
| 151 | return Dir(get_configured_path(PATH_STATE_DIR)) |
| 152 | |
| 153 | def get_suites_dir(): |
| 154 | return Dir(get_configured_path(PATH_SUITES_DIR)) |
| 155 | |
| 156 | def get_scenarios_dir(): |
| 157 | return Dir(get_configured_path(PATH_SCENARIOS_DIR)) |
| 158 | |
| 159 | def read(path, validation_schema=None, if_missing_return=False): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 160 | log.ctx(path) |
| 161 | if not os.path.isfile(path) and if_missing_return is not False: |
| 162 | return if_missing_return |
| 163 | with open(path, 'r') as f: |
| 164 | config = yaml.safe_load(f) |
| 165 | config = _standardize(config) |
| 166 | if validation_schema: |
| 167 | schema.validate(config, validation_schema) |
| 168 | return config |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 169 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 170 | def write(path, config): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 171 | log.ctx(path) |
| 172 | with open(path, 'w') as f: |
| 173 | f.write(tostr(config)) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 174 | |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 175 | def tostr(config): |
| 176 | return _tostr(_standardize(config)) |
| 177 | |
| 178 | def _tostr(config): |
| 179 | return yaml.dump(config, default_flow_style=False) |
| 180 | |
| 181 | def _standardize_item(item): |
Pau Espin Pedrol | 7691f2d | 2020-02-18 12:12:01 +0100 | [diff] [blame] | 182 | if item is None: |
| 183 | return None |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 184 | if isinstance(item, (tuple, list)): |
| 185 | return [_standardize_item(i) for i in item] |
| 186 | if isinstance(item, dict): |
| 187 | return dict([(key.lower(), _standardize_item(val)) for key,val in item.items()]) |
| 188 | return str(item) |
| 189 | |
| 190 | def _standardize(config): |
| 191 | config = yaml.safe_load(_tostr(_standardize_item(config))) |
| 192 | return config |
| 193 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 194 | def get_defaults(for_kind): |
Pau Espin Pedrol | a2e81f5 | 2020-03-11 20:07:23 +0100 | [diff] [blame] | 195 | defaults = read_config_file(DEFAULTS_CONF, if_missing_return={}) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 196 | return defaults.get(for_kind, {}) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 197 | |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 198 | class Scenario(log.Origin, dict): |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 199 | def __init__(self, name, path, param_list=[]): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 200 | super().__init__(log.C_TST, name) |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 201 | self.path = path |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 202 | self.param_list = param_list |
| 203 | |
Pau Espin Pedrol | 20a4911 | 2020-04-01 19:48:41 +0200 | [diff] [blame] | 204 | @classmethod |
| 205 | def count_cont_char_backward(cls, str, before_pos, c): |
| 206 | n = 0 |
| 207 | i = before_pos - 1 |
| 208 | while i >= 0: |
| 209 | if str[i] != c: |
| 210 | break |
| 211 | n += 1 |
| 212 | i -= 1 |
| 213 | return n |
| 214 | |
| 215 | @classmethod |
| 216 | def split_scenario_parameters(cls, str): |
| 217 | cur_pos = 0 |
| 218 | param_li = [] |
| 219 | saved = '' |
| 220 | # Split into a list, but we want to escape '\,' to avoid splitting parameters containing commas. |
| 221 | while True: |
| 222 | prev_pos = cur_pos |
| 223 | cur_pos = str.find(',', prev_pos) |
| 224 | if cur_pos == -1: |
| 225 | param_li.append(str[prev_pos:]) |
| 226 | break |
| 227 | if cur_pos == 0: |
| 228 | param_li.append('') |
| 229 | elif cur_pos != 0 and str[cur_pos - 1] == '\\' and cls.count_cont_char_backward(str, cur_pos, '\\') % 2 == 1: |
| 230 | saved += str[prev_pos:cur_pos - 1] + ',' |
| 231 | else: |
| 232 | param_li.append(saved + str[prev_pos:cur_pos]) |
| 233 | saved = '' |
| 234 | cur_pos += 1 |
| 235 | i = 0 |
| 236 | # Also escape '\\' -> '\' |
| 237 | while i < len(param_li): |
| 238 | param_li[i] = param_li[i].replace('\\\\', '\\') |
| 239 | i += 1 |
| 240 | return param_li |
| 241 | |
| 242 | @classmethod |
| 243 | def from_param_list_str(cls, name, path, param_list_str): |
| 244 | param_list = cls.split_scenario_parameters(param_list_str) |
| 245 | return cls(name, path, param_list) |
| 246 | |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 247 | def read_from_file(self, validation_schema): |
| 248 | with open(self.path, 'r') as f: |
| 249 | config_str = f.read() |
| 250 | if len(self.param_list) != 0: |
| 251 | param_dict = {} |
| 252 | i = 1 |
| 253 | for param in self.param_list: |
| 254 | param_dict['param' + str(i)] = param |
| 255 | i += 1 |
| 256 | self.dbg(param_dict=param_dict) |
| 257 | config_str = template.render_strbuf_inline(config_str, param_dict) |
| 258 | config = yaml.safe_load(config_str) |
| 259 | config = _standardize(config) |
| 260 | if validation_schema: |
| 261 | schema.validate(config, validation_schema) |
| 262 | self.update(config) |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 263 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 264 | def get_scenario(name, validation_schema=None): |
| 265 | scenarios_dir = get_scenarios_dir() |
| 266 | if not name.endswith('.conf'): |
| 267 | name = name + '.conf' |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 268 | is_parametrized_file = '@' in name |
| 269 | param_list = [] |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 270 | path = scenarios_dir.child(name) |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 271 | if not is_parametrized_file: |
| 272 | if not os.path.isfile(path): |
| 273 | raise RuntimeError('No such scenario file: %r' % path) |
Pau Espin Pedrol | 20a4911 | 2020-04-01 19:48:41 +0200 | [diff] [blame] | 274 | sc = Scenario(name, path) |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 275 | else: # parametrized scenario: |
| 276 | # Allow first matching complete matching names (eg: scenario@param1,param2.conf), |
| 277 | # this allows setting specific content in different files for specific values. |
| 278 | if not os.path.isfile(path): |
| 279 | # get "scenario@.conf" from "scenario@param1,param2.conf": |
| 280 | prefix_name = name[:name.index("@")+1] + '.conf' |
| 281 | path = scenarios_dir.child(prefix_name) |
| 282 | if not os.path.isfile(path): |
| 283 | raise RuntimeError('No such scenario file: %r (nor %s)' % (path, name)) |
| 284 | # At this point, we have existing file path. Let's now scrap the parameter(s): |
| 285 | # get param1,param2 str from scenario@param1,param2.conf |
| 286 | param_list_str = name.split('@', 1)[1][:-len('.conf')] |
Pau Espin Pedrol | 20a4911 | 2020-04-01 19:48:41 +0200 | [diff] [blame] | 287 | sc = Scenario.from_param_list_str(name, path, param_list_str) |
Pau Espin Pedrol | 6ed3012 | 2020-02-27 17:03:15 +0100 | [diff] [blame] | 288 | sc.read_from_file(validation_schema) |
Your Name | 44af341 | 2017-04-13 03:11:59 +0200 | [diff] [blame] | 289 | return sc |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 290 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 291 | def add(dest, src): |
| 292 | if is_dict(dest): |
| 293 | if not is_dict(src): |
| 294 | raise ValueError('cannot add to dict a value of type: %r' % type(src)) |
| 295 | |
| 296 | for key, val in src.items(): |
| 297 | dest_val = dest.get(key) |
| 298 | if dest_val is None: |
| 299 | dest[key] = val |
| 300 | else: |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 301 | log.ctx(key=key) |
| 302 | add(dest_val, val) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 303 | return |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 304 | if is_list(dest): |
| 305 | if not is_list(src): |
| 306 | raise ValueError('cannot add to list a value of type: %r' % type(src)) |
| 307 | dest.extend(src) |
| 308 | return |
| 309 | if dest == src: |
| 310 | return |
| 311 | raise ValueError('cannot add dicts, conflicting items (values %r and %r)' |
| 312 | % (dest, src)) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 313 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 314 | def combine(dest, src): |
| 315 | if is_dict(dest): |
| 316 | if not is_dict(src): |
| 317 | raise ValueError('cannot combine dict with a value of type: %r' % type(src)) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 318 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 319 | for key, val in src.items(): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 320 | log.ctx(key=key) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 321 | dest_val = dest.get(key) |
| 322 | if dest_val is None: |
| 323 | dest[key] = val |
| 324 | else: |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 325 | combine(dest_val, val) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 326 | return |
| 327 | if is_list(dest): |
| 328 | if not is_list(src): |
| 329 | raise ValueError('cannot combine list with a value of type: %r' % type(src)) |
Pau Espin Pedrol | 43737da | 2017-08-28 14:04:07 +0200 | [diff] [blame] | 330 | # Validate that all elements in both lists are of the same type: |
| 331 | t = util.list_validate_same_elem_type(src + dest) |
| 332 | if t is None: |
| 333 | return # both lists are empty, return |
| 334 | # For lists of complex objects, we expect them to be sorted lists: |
| 335 | if t in (dict, list, tuple): |
| 336 | for i in range(len(dest)): |
| 337 | log.ctx(idx=i) |
| 338 | src_it = src[i] if i < len(src) else util.empty_instance_type(t) |
| 339 | combine(dest[i], src_it) |
| 340 | for i in range(len(dest), len(src)): |
| 341 | log.ctx(idx=i) |
| 342 | dest.append(src[i]) |
| 343 | else: # for lists of basic elements, we handle them as unsorted sets: |
| 344 | for elem in src: |
| 345 | if elem not in dest: |
| 346 | dest.append(elem) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 347 | return |
| 348 | if dest == src: |
| 349 | return |
| 350 | raise ValueError('cannot combine dicts, conflicting items (values %r and %r)' |
| 351 | % (dest, src)) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 352 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 353 | def overlay(dest, src): |
| 354 | if is_dict(dest): |
| 355 | if not is_dict(src): |
| 356 | raise ValueError('cannot combine dict with a value of type: %r' % type(src)) |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 357 | |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 358 | for key, val in src.items(): |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 359 | log.ctx(key=key) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 360 | dest_val = dest.get(key) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 361 | dest[key] = overlay(dest_val, val) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 362 | return dest |
| 363 | if is_list(dest): |
| 364 | if not is_list(src): |
| 365 | raise ValueError('cannot combine list with a value of type: %r' % type(src)) |
Pau Espin Pedrol | 2753204 | 2017-09-15 15:31:52 +0200 | [diff] [blame] | 366 | copy_len = min(len(src),len(dest)) |
| 367 | for i in range(copy_len): |
Pau Espin Pedrol | ebced95 | 2017-08-28 17:21:34 +0200 | [diff] [blame] | 368 | log.ctx(idx=i) |
Neels Hofmeyr | 1a7a3f0 | 2017-06-10 01:18:27 +0200 | [diff] [blame] | 369 | dest[i] = overlay(dest[i], src[i]) |
Pau Espin Pedrol | 2753204 | 2017-09-15 15:31:52 +0200 | [diff] [blame] | 370 | for i in range(copy_len, len(src)): |
| 371 | dest.append(src[i]) |
Neels Hofmeyr | 3531a19 | 2017-03-28 14:30:28 +0200 | [diff] [blame] | 372 | return dest |
| 373 | return src |
Pau Espin Pedrol | 802dfe5 | 2017-09-12 13:43:40 +0200 | [diff] [blame] | 374 | |
| 375 | def replicate_times(d): |
Pau Espin Pedrol | 0b30279 | 2017-09-10 16:33:10 +0200 | [diff] [blame] | 376 | ''' |
| 377 | replicate items that have a "times" > 1 |
| 378 | |
| 379 | 'd' is a dict matching WANT_SCHEMA, which is the same as |
| 380 | the RESOURCES_SCHEMA, except each entity that can be reserved has a 'times' |
| 381 | field added, to indicate how many of those should be reserved. |
| 382 | ''' |
Pau Espin Pedrol | 802dfe5 | 2017-09-12 13:43:40 +0200 | [diff] [blame] | 383 | d = copy.deepcopy(d) |
| 384 | for key, item_list in d.items(): |
Pau Espin Pedrol | 2605034 | 2017-09-12 15:02:25 +0200 | [diff] [blame] | 385 | idx = 0 |
| 386 | while idx < len(item_list): |
| 387 | item = item_list[idx] |
| 388 | times = int(item.pop('times', 1)) |
| 389 | for j in range(1, times): |
| 390 | item_list.insert(idx + j, copy.deepcopy(item)) |
| 391 | idx += times |
Pau Espin Pedrol | 802dfe5 | 2017-09-12 13:43:40 +0200 | [diff] [blame] | 392 | return d |
| 393 | |
Neels Hofmeyr | dae3d3c | 2017-03-28 12:16:58 +0200 | [diff] [blame] | 394 | # vim: expandtab tabstop=4 shiftwidth=4 |